DcuplAppLoader API

npm install @dcupl/loader

import { Dcupl } from '@dcupl/core';
import { DcuplAppLoader } from '@dcupl/loader';

// projectId and apiKey belong to the dcupl instance, the loader reads them from there
const dcupl = new Dcupl({
  config: { projectId: 'my-project', apiKey: 'my-api-key' },
});
const loader = new DcuplAppLoader();

// Register the loader first: config.fetch() and process() need it
dcupl.loaders.add(loader, 'main');

// Fetch the loader configuration (dcupl.lc.json) from the dcupl CDN
await loader.config.fetch();

// Download the models and data listed in the configuration
await loader.process();

// Build models, references and indices
await dcupl.init();

type ProjectConfiguration = {
  resources: Resource[]; // see Resource Types below
  applications?: {
    key: string;
    name?: string;
    description?: string;
    resourceTags?: (string | string[])[];
    variables?: Variable[];
  }[];
  environments?: {
    key: string;
    description?: string;
    variables?: Variable[];
    headers?: HttpConfig[]; // request headers for resource downloads
    queryParams?: HttpConfig[]; // query parameters appended to resource urls
  }[];
  options?: ProcessOptions['options']; // see ProcessOptions below
};

type Variable = { key: string; value: string | undefined; description?: string };

type HttpConfig = {
  key: string;
  value: string; // may contain ${variables}
  tags?: string[]; // only for resources sharing one of these tags; without tags: every resource (see below for headers)
  condition?: { applyIfVariableHasValue?: string }; // only when this variable has a non-empty value
};

type FetchOptions = {
  baseUrl?: string; // Where the config file lives (default: your project's draft version on the dcupl CDN)
  loaderFileName?: string; // Config file name (default: 'dcupl.lc.json')
  headers?: { key: string; value: string }[]; // Request headers, used when fetching from a custom baseUrl
  skipApply?: boolean; // Return the config without applying it to the loader
};

// Fetch from the dcupl CDN (projectId and apiKey come from new Dcupl({ config }))
await loader.config.fetch();

// Fetch from custom URL
await loader.config.fetch({
  baseUrl: 'https://my-cdn.example.com/config',
  loaderFileName: 'dcupl.loader.json',
  headers: [{ key: 'Authorization', value: 'Bearer my-token' }],
});

// Get current config
const config = loader.config.get();
console.log(config?.resources.length);

// Set config manually (models are loaded from 'model' resources too)
loader.config.set({
  resources: [
    { type: 'model', url: '/models/product.dcupl.json' },
    { key: 'products', type: 'data', url: '/data/products.json', model: 'Product' },
  ],
});

type Resource = ModelResource | DataResource | TransformerResource | CustomOperatorResource | CustomScriptResource;

// Fields shared by every resource type
type ResourceBase = {
  key?: string; // Optional; used by resources.get()/delete() and as the plan id
  url: string; // May contain ${variables}
  type: 'model' | 'data' | 'transformer' | 'operator' | 'script';
  tags?: string[]; // For filtering resources and matching transformers
};

type ModelResource = ResourceBase & { type: 'model' }; // the file holds a model definition, an array of them, or a view definition

type DataResource = ResourceBase & {
  type: 'data';
  model: string; // Target model
  options?: {
    updateType?: 'set' | 'update' | 'upsert' | 'remove';
    keyProperty?: string | string[];
    keyPropertySeparator?: string;
    autoGenerateKey?: boolean;
    autoGenerateProperties?: boolean | { deep?: boolean; sampleSize?: number };
    csvParserOptions?: {
      delimiter?: string;
      quoteChar?: string;
      escapeChar?: string;
      commentChar?: string;
    };
  };
};

type TransformerResource = ResourceBase & {
  type: 'transformer';
  applyTo?: string[]; // Tags of the data resources it runs on
  transformerType?: 'rawFileTransformer' | 'parsedFileTransformer'; // Default: 'parsedFileTransformer'
};

type CustomOperatorResource = ResourceBase & {
  type: 'operator';
  operator: string; // The operator name usable in queries
};

type CustomScriptResource = ResourceBase & {
  type: 'script';
  scriptKey: string; // The key the script is registered under (see /sdk/guides/scripts)
};

type ResourcePlan = {
  scheduled: ResourceItem[]; // what process() would load — tag-filtered, urls resolved
  skipped: SkippedResource[]; // declared but not scheduled for this run
  variables: Record<string, any>; // the variables the scheduled urls were resolved with
  run: RunContext; // the applicationKey / resourceTags / environmentKeys used
};

type SkippedResource = {
  resource: Resource;
  reason: 'no-tags' | 'tag-mismatch';
  id?: string;
};

// Preview what a run would load, without loading it
const plan = loader.resources.plan({ applicationKey: 'catalog', resourceTags: ['critical'] });

console.log(plan.scheduled.map((r) => r.id));
plan.skipped.forEach((s) => console.log(`${s.resource.url} skipped: ${s.reason}`));

// Catch a misconfigured variable before it becomes a download failure
const broken = plan.scheduled.filter((r) => r.unresolvedVariables?.length);

// Get all resources
const resources = loader.resources.getAll();

// Get resources by tag
const criticalResources = loader.resources.getWithTag('critical');

// Add a new data resource
loader.resources.add({
  key: 'products-update',
  type: 'data',
  url: '/api/products/latest',
  model: 'Product',
  options: { updateType: 'upsert' },
  tags: ['live-data'],
});

// Delete a resource
loader.resources.delete('products-update');

${dataUrl}/products.csv  ->  ${cdnBaseUrl}/${projectId}/${version}/data/products.csv
${baseUrl}/data/products.csv  ->  https://cdn.dcupl.com/files/projects/<projectId>/draft/data/products.csv

// Built-in default variables (projectId, apiKey, version, baseUrl, ...)
loader.variables.default.get(key);
loader.variables.default.getAll();

// Environment variables (from config environments)
loader.variables.environment.get(env, key);
loader.variables.environment.getAll(env);

// Process variables (passed during processing)
loader.variables.process.get(key);
loader.variables.process.getAll();

// Global variables (set at runtime)
loader.variables.global.get(key);
loader.variables.global.getAll();
loader.variables.global.set(key, value);
loader.variables.global.delete(key);

// Get API URL variable, resolved for the production environment
const apiUrl = loader.variables.get('API_URL', 'production');
console.log(apiUrl?.value);

// Get all variables for production environment
const prodVars = loader.variables.getAll('production');

// Get variables as object for easy access
const vars = loader.variables.getAllAsObject('production');
console.log(vars.API_URL);

// Set global variable at runtime
loader.variables.global.set('AUTH_TOKEN', 'Bearer xyz');

// Use in resource URLs with interpolation
// URL: "${API_URL}/products" becomes "https://api.example.com/products"

loader.config.set({
  resources: [
    { type: 'model', url: '${baseUrl}/models/product.dcupl.json' },
    {
      type: 'transformer',
      url: '${baseUrl}/transformers/normalize-prices.js',
      applyTo: ['products'], // runs on data resources tagged 'products'
      transformerType: 'parsedFileTransformer', // default
    },
    { type: 'data', url: '${baseUrl}/data/products.csv', model: 'Product', tags: ['products'] },
  ],
});

return (resource, items) =>
  items.map((item) => ({
    ...item,
    price: Number(item.price) || 0,
    currency: item.currency || 'USD',
  }));

type TransformerConfigObject = {
  key: string; // A transformer loaded from a resource is registered under its resolved url
  applyTo: string[]; // Tags of the data resources it runs on
  type: 'rawFileTransformer' | 'parsedFileTransformer';
  transformer: (resource: AppLoaderConfiguration.Resource, input: any) => any;
};

// Supply the function for a declared transformer resource instead of downloading the file
loader.transformers.set({
  key: 'https://cdn.example.com/transformers/normalize-prices.js', // the resource's resolved url
  applyTo: ['products'],
  type: 'parsedFileTransformer',
  transformer: (resource, items) =>
    items.map((item: Record<string, any>) => ({ ...item, currency: item.currency || 'USD' })),
});

// Get registered transformers for a tag (note the spelling: appyTo)
const productTransformers = loader.transformers.get({
  appyTo: ['products'],
});

// Remove a transformer
loader.transformers.remove('https://cdn.example.com/transformers/normalize-prices.js');

const loaderErrors = dcupl.quality.errors.summary().filter((group) => group.type === 'loader');
// e.g. [{ key: 'Product::::TransformerError', model: 'Product', errorGroup: 'ResourceError', errorType: 'TransformerError', type: 'loader', totalCount: 1, ... }]

await loader.process(options?: ProcessOptions); // Promise<boolean>

type ProcessOptions = {
  applicationKey?: string; // Select an application preset (its resourceTags and variables), see Applications
  environmentKeys?: string[]; // Select environment configurations
  resourceTags?: (string | string[])[]; // Filter resources by tags
  variables?: { key: string; value: string | undefined; description?: string }[]; // Override variables
  options?: {
    defaultDataContainerUpdateType?: 'set' | 'update' | 'upsert' | 'remove';
    csvParserOptions?: CsvParserOptions;
    missingModelHandling?: {
      // default: true — a data resource whose model has no definition gets a model with inferred properties
      autoGenerateProperties?: boolean | { deep?: boolean; sampleSize?: number };
    };
  };
};

// Process with specific environment
await loader.process({
  environmentKeys: ['production'],
});

// Process only tagged resources
await loader.process({
  resourceTags: ['critical', 'models'],
});

// Process with variable overrides
await loader.process({
  variables: [{ key: 'API_URL', value: 'https://staging-api.example.com' }],
});

{
  "applications": [
    { "key": "catalog", "resourceTags": ["catalog"] },
    { "key": "default", "name": "Default" }
  ],
  "resources": [
    { "type": "model", "url": "${baseUrl}/models/product.dcupl.json", "tags": ["catalog"] },
    { "type": "data", "url": "${baseUrl}/data/products.csv", "model": "Product", "tags": ["catalog"] },
    { "type": "model", "url": "${baseUrl}/models/order.dcupl.json", "tags": ["orders"] },
    { "type": "data", "url": "${baseUrl}/data/orders.csv", "model": "Order", "tags": ["orders"] }
  ]
}

await loader.process({ applicationKey: 'catalog' }); // Product only
await loader.process({ applicationKey: 'default' }); // no resourceTags: every resource
await loader.process(); // no application: every resource

import type { AppLoaderProgress } from '@dcupl/loader';

const unsubscribe = loader.on((progress: AppLoaderProgress.ProgressItem) => {
  console.log(`${progress.processed}/${progress.total} resources processed`);
});

// Later: unsubscribe
unsubscribe();

type ProgressItem = {
  total: number; // resources scheduled for this run
  processed: number;
  remaining: number; // 0 once the run is finished
  progress: number; // percentage, 0-100
  context: string; // the type of the resource just handled: 'model', 'data', 'transformer', ...
  currentItem: ResourceItem; // the resource this update is about (undefined in the first update)

  totalItems: ResourceItem[]; // scheduled — tag-filtered, urls resolved
  processedItems: ResourceItem[];
  downloadedItems: ResourceItem[];
  failedItems: ResourceItem[];
  remainingItems: ResourceItem[];

  skippedItems?: SkippedResource[]; // declared but never scheduled, with a reason
  run?: RunContext; // applicationKey / resourceTags / environmentKeys
};

loader.on((progress) => {
  if (progress.remaining !== 0) return;

  for (const failure of progress.failedItems) {
    // status is only set when a response arrived (not on a network error or a bad url)
    console.error(`${failure.key ?? failure.url} failed with ${failure.status ?? 'no response'}`);
  }
});

import { Dcupl } from '@dcupl/core';
import { DcuplAppLoader } from '@dcupl/loader';

// Runs in the browser (navigator.language below)
async function initializeDcupl() {
  const dcupl = new Dcupl({
    config: {
      projectId: 'my-project',
      apiKey: 'my-read-only-api-key', // a scoped read-only key: it ships to every visitor
    },
  });
  const loader = new DcuplAppLoader();

  // Register with dcupl before fetching
  dcupl.loaders.add(loader, 'main');

  // Track progress
  loader.on((progress) => {
    const current = progress.currentItem?.key ?? progress.currentItem?.url;
    console.log(`Loading: ${current ?? 'starting'} (${progress.processed}/${progress.total})`);
  });

  // Fetch configuration
  await loader.config.fetch();

  // Set runtime variables, usable as ${USER_LOCALE} in resource urls
  loader.variables.global.set('USER_LOCALE', navigator.language);

  // Download models and data with the production environment
  await loader.process({
    environmentKeys: ['production'],
  });

  // Initialize
  await dcupl.init();

  return dcupl;
}