import { promises as fs } from "fs";
import { Spinner } from "@topcli/spinner";
import * as ts from "typescript";
import { QUERY_API_DEFINITIONS } from "./consts.js";
import { slugToPascal, getNamedTypePath, attachImport } from "./utils.js";
import getRadixTypeDefinitionAsJson from "./get-radix-type-as-json.js";

// main(() => {});

export default async function main(patchExpandables) {
  const PRIMITIVES = [
    "string",
    "boolean",
    "number",
    "object",
    "array",
    "undefined",
    "null",
  ];

  const spinner = new Spinner().start("Generating UI Kit snippets");

  const uiKitPath = "node_modules/@propeldata/ui-kit/dist/esm";
  const uiKitPathComponents = "node_modules/@propeldata/ui-kit/dist/esm";

  const snippetsPath = "./snippets";
  const uiKitSnippetsPath = `${snippetsPath}/ui-kit`;

  await fs.rm(uiKitSnippetsPath, { recursive: true, force: true });

  const tsFiles = (await fs.readdir(uiKitPath, { recursive: true }))
    .map((dir) => `${uiKitPath}/${dir}`)
    .filter((file) => file.endsWith(".types.d.ts"));

  const fullProgram = ts.createProgram(tsFiles, { allowJs: true });

  const { scalars } = QUERY_API_DEFINITIONS;

  /**
   * Finds a symbol in a source file
   *
   * @param {string} symbolName
   * @returns {Array<ts.Node['members']>}
   */
  function findSymbol(symbolName) {
    const checker = fullProgram.getTypeChecker();

    // Iterate through all source files in the program
    for (const sourceFile of fullProgram.getSourceFiles()) {
      // Check if the source file is part of the program
      if (!tsFiles.some((name) => sourceFile.fileName.includes(name))) {
        continue;
      }

      // Find the symbol in the current source file
      const symbol =
        checker
          .getSymbolsInScope(sourceFile, ts.SymbolFlags.Interface)
          .find((s) => s.getName() === symbolName) ??
        checker
          .getSymbolsInScope(sourceFile, ts.SymbolFlags.TypeAlias)
          .find((s) => s.getName() === symbolName);

      if (symbol != null) {
        return symbol;
      }
    }

    return [];
  }

  /**
   * Extracts the members of a node's heritage clause
   *
   * @param {ts.Node} node
   * @param {ts.Printer} printer
   * @param {ts.SourceFile} sourceFile
   * @returns {{ members: Array<ts.Node['members']>, isThemeProvider: boolean }}
   */
  function extractHeritageMembers(node, printer, sourceFile) {
    let members = [];

    let omittedMembers = [];

    if (node.heritageClauses) {
      const clauseType = node.heritageClauses[0].types[0];

      const heritageClauseFromCompound = clauseType.typeArguments?.[0];
      const heritageClauseFromSimple = clauseType;

      const isOmit =
        clauseType.kind === ts.SyntaxKind.ExpressionWithTypeArguments &&
        clauseType.expression.escapedText === "Omit";

      const heritageType = printer.printNode(
        ts.EmitHint.Unspecified,
        heritageClauseFromCompound ?? heritageClauseFromSimple,
        sourceFile,
      );

      if (heritageType === "ThemeSettingProps") {
        return { members, isThemeProvider: true };
      }

      const symbol = findSymbol(heritageType);

      if (symbol.declarations?.[0]?.members != null) {
        if (isOmit) {
          omittedMembers = clauseType.typeArguments.slice(1).map((m) => m?.literal?.text);
        }

        members.push(
          ...symbol.declarations[0].members.filter(
            (m) => !omittedMembers.includes(m.name.text),
          ),
        );
      }

      if (symbol.declarations?.[0]?.heritageClauses != null) {
        members.push(
          ...extractHeritageMembers(symbol.declarations[0], printer, sourceFile).members,
        );
      }
    }

    return { members, isThemeProvider: false };
  }

  /**
   * Prints out particular nodes from a source file
   *
   * @param {string} file a path to a file
   * @returns {Array<{name: string, type: string, description: string, fields: Array<{name: string, type: string, description: string, isDeprecated: boolean, path: string}>, isThemeProvider: boolean}>}
   */
  function extract(file) {
    let program = ts.createProgram([file], { allowJs: true });
    const sourceFile = program.getSourceFile(file);

    const printer = ts.createPrinter({ newLine: ts.NewLineKind.LineFeed });

    const definitions = [];

    ts.forEachChild(sourceFile, (node) => {
      if (ts.isInterfaceDeclaration(node) || ts.isTypeAliasDeclaration(node)) {
        let { members: heritageMembers, isThemeProvider } = extractHeritageMembers(
          node,
          printer,
          sourceFile,
        );

        const nodeMembers = (node.members ?? []).sort((a, b) => {
          if (a.name?.text === "query") return -1;
          if (b.name?.text === "query") return 1;
          return 0;
        });

        definitions.push({
          name: node.name.text,
          fields:
            [...(nodeMembers ?? []), ...(heritageMembers ?? [])]
              .map((member) => {
                if (member.name == null) return null;

                const isDeprecated =
                  member.jsDoc?.[0].tags?.some(
                    (tag) => tag.tagName.escapedText === "deprecated",
                  ) ?? false;

                const description =
                  member.jsDoc?.[0].comment ?? member.jsDoc?.[0].tags?.[0].comment ?? "";

                return {
                  name: member.name?.text,
                  type: printer.printNode(
                    ts.EmitHint.Unspecified,
                    member.type,
                    sourceFile,
                  ),
                  description,
                  isDeprecated,
                };
              })
              .filter((field) => field != null) ?? [],
          type:
            node.type &&
            printer.printNode(ts.EmitHint.Unspecified, node.type, sourceFile),
          path: file,
          isThemeProvider: isThemeProvider,
        });
      }
    });

    for (const definition of definitions) {
      if (definition.fields.length === 0) {
        // Find in radix
        try {
          const radixTypeDefinition = getRadixTypeDefinitionAsJson(definition.name);
          definition.fields = Object.entries(radixTypeDefinition).map(([name, type]) => ({
            name,
            type,
            description: "",
            isDeprecated: false,
          }));

          definition.type = null;
        } catch {
          // Not found
        }
      }
    }

    if (!definitions.length) {
      console.log("No declarations found");
    }

    return definitions;
  }

  const componentsDirectory = await fs.readdir(uiKitPathComponents);

  await Promise.all(
    componentsDirectory.map((path) => {
      return new Promise(async (resolve) => {
        await generateSnippets(`${uiKitPathComponents}/${path}`);
        resolve();
      });
    }),
  );

  spinner.succeed("UI Kit snippets generated successfully");

  /**
   * Receives a file name or directory name and generates snippets for the file
   * @param {string} path
   */
  async function generateSnippets(path) {
    const fileStat = await fs.stat(path);
    const isFile = fileStat.isFile();

    const fileName = path.split("/").pop();

    if (isFile && fileName.endsWith(".types.d.ts")) {
      const definitions = extract(path);

      await Promise.all(
        definitions.map((definition) => {
          return new Promise(async (resolve) => {
            const definitionName = definition.name;
            const fields = definition.fields;

            const deprecatedParamFields = [];

            const imports = new Set();

            if (definition.type != null) {
              fields.unshift({
                name: definitionName,
                type: definition.type,
                description: definition.description,
              });
            }

            let snippetContent = `\n${fields
              .map((field) => {
                const isDeprecated = field.isDeprecated;
                const withDeprecated = isDeprecated ? " deprecated" : "";

                const namedTypes = field.type
                  .split("|")
                  .map((t) =>
                    t
                      .replaceAll("(", "")
                      .replaceAll(")", "")
                      .replaceAll("[", "")
                      .replaceAll("]", "")
                      .replaceAll("?", "")
                      .trim(),
                  );

                let withExpandable = "";

                namedTypes.forEach((namedType) => {
                  const namedTypePath = getNamedTypePath(namedType, definitions);

                  if (namedTypePath == null) {
                    return;
                  }

                  const isPrimitiveOrScalar =
                    PRIMITIVES.includes(namedType) || scalars.includes(namedType);
                  const isRecursive = namedType === slugToPascal(fileName);

                  if (!isPrimitiveOrScalar && !isRecursive) {
                    imports.add(namedType);

                    withExpandable += `\n\t<Expandable title="${namedType}">\n\t\t<${namedType} />\n\t</Expandable>\n`;
                  }
                });

                const paramField = `<ParamField path="${field.name}" type={\`${
                  field.type
                }\`}${withDeprecated}>\n\t${
                  field.description ?? ""
                }\n${withExpandable}</ParamField>\n`;

                if (isDeprecated) {
                  deprecatedParamFields.push(paramField);
                  return null;
                }

                return paramField;
              })
              .filter((field) => field != null)
              .join("\n")}`;

            if (definition.isThemeProvider) {
              snippetContent = attachThemeProvider(snippetContent);
            }

            if (deprecatedParamFields.length > 0) {
              snippetContent += `\n\n<Expandable title="Deprecated Fields">\n${Array.from(
                deprecatedParamFields,
              ).join("\n")}\n</Expandable>\n`;
            }

            imports.forEach((definitionName) => {
              snippetContent = attachImport(snippetContent, definitionName, definitions);
            });

            snippetContent = patchExpandables(snippetContent);

            await fs.mkdir(
              `${uiKitSnippetsPath}/${fileName.replace(".types.d.ts", "")}`,
              { recursive: true },
            );
            const filePath = `${uiKitSnippetsPath}/${fileName.replace(
              ".types.d.ts",
              "",
            )}/${definitionName}.mdx`;
            await fs.writeFile(filePath, snippetContent);

            resolve();
          });
        }),
      );
    } else if (!isFile) {
      const componentsDirectory = (await fs.readdir(path)).map(
        (name) => `${path}/${name}`,
      );
      componentsDirectory.forEach(async (path) => {
        await generateSnippets(path);
      });
    }
  }

  function attachThemeProvider(snippetContent) {
    let content = `import ThemeSettingProps from '/snippets/ui-kit/theme/ThemeSettingProps.mdx'\n${snippetContent}`;

    content += `\n<Expandable title="Theme Provider Props">\n\t<ThemeSettingProps />\n</Expandable>\n`;

    return content;
  }
}


import { promises as fs } from "fs";
import { parse, Kind } from "graphql";

export const QUERY_API_DEFINITIONS = await getApiDefinitions('./node_modules/@propeldata/config-management-api-definitions/dist/query-api.graphql')
export const MANAGEMENT_API_DEFINITIONS = await getApiDefinitions('./node_modules/@propeldata/config-management-api-definitions/dist/public.graphql')

/**
 * @param {string} path
 */
async function getApiDefinitions(path) {
  const apiSchema = await fs.readFile(
    path,
    "utf8"
  );

  const parsedSchema = parse(apiSchema);

  const enums = parsedSchema.definitions.filter(
    (definition) => definition.kind === Kind.ENUM_TYPE_DEFINITION
  );
  
  const inputs = parsedSchema.definitions.filter(
    (definition) => definition.kind === Kind.INPUT_OBJECT_TYPE_DEFINITION
  );
  
  const types = parsedSchema.definitions.filter(
    (definition) => definition.kind === Kind.OBJECT_TYPE_DEFINITION
  );

  const unions = parsedSchema.definitions.filter(
    (definition) => definition.kind === Kind.UNION_TYPE_DEFINITION
  );
  
  const queries = parsedSchema.definitions.filter(
    (definition) => definition.name?.value === 'Query'
  );

  const mutations = parsedSchema.definitions.filter(
    (definition) => definition.name?.value === 'Mutation'
  );

  const scalars = parsedSchema.definitions.filter(
    (definition) => definition.kind === Kind.SCALAR_TYPE_DEFINITION
  );

  const scalarsNames = scalars.map(s => s.name?.value ?? s.name);

  return { enums, inputs, types, scalars, scalarsNames, unions, queries, mutations }
}

import { MANAGEMENT_API_DEFINITIONS, QUERY_API_DEFINITIONS } from "./consts.js"

/**
 * @param {string} fileContent 
 * @param {string} definitionName 
 * @param {Array<{name: string, type: string, description: string, fields: Array<{name: string, type: string, description: string, isDeprecated: boolean}>}>} uiKitDefinitions 
 */
export function attachImport(fileContent, definitionName, uiKitDefinitions) {
  const statement = `import ${definitionName} from '${getNamedTypePath(definitionName, uiKitDefinitions)}'`

  return `${statement}\n${fileContent}`
}

/**
 * @param {string} definitionName 
 * @returns {Array<'enums' | 'inputs' | 'types' | null, 'query-apis' | 'management-apis'>}
 */
export function getDefinitionType(definitionName) {
  const { enums: queryApiEnums, inputs: queryApiInputs, types: queryApiTypes, unions: queryApiUnions, queries: queryApiQueries, mutations: queryApiMutations, scalars: queryApiScalars } = QUERY_API_DEFINITIONS
  const { enums: managementApiEnums, inputs: managementApiInputs, types: managementApiTypes, unions: managementApiUnions, queries: managementApiQueries, mutations: managementApiMutations, scalars: managementApiScalars } = MANAGEMENT_API_DEFINITIONS

  if (queryApiEnums.map(e => e.name.value).includes(definitionName)) {
    return ["enums", 'query-apis']
  }
  if (queryApiInputs.map(i => i.name.value).includes(definitionName)) {
    return ["inputs", 'query-apis']
  }
  if (queryApiTypes.map(t => t.name.value).includes(definitionName)) {
    return ["types", 'query-apis']
  }
  if (queryApiUnions.map(u => u.name.value).includes(definitionName)) {
    return ["unions", 'query-apis']
  }
  if (queryApiScalars.map(s => s.name.value).includes(definitionName)) {
    return ["scalars", 'query-apis']
  }
  if (queryApiQueries.map(q => q.name.value).includes(definitionName)) {
    return ["queries", 'query-apis']
  }
  if (queryApiMutations.map(m => m.name.value).includes(definitionName)) {
    return ["mutations", 'query-apis']
  }
  
  if (managementApiEnums.map(e => e.name.value).includes(definitionName)) {
    return ["enums", 'management-apis']
  }
  if (managementApiInputs.map(i => i.name.value).includes(definitionName)) {
    return ["inputs", 'management-apis']
  }
  if (managementApiTypes.map(t => t.name.value).includes(definitionName)) {
    return ["types", 'management-apis']
  }
  if (managementApiUnions.map(u => u.name.value).includes(definitionName)) {
    return ["unions", 'management-apis']
  }
  if (managementApiScalars.map(s => s.name.value).includes(definitionName)) {
    return ["scalars", 'management-apis']
  }
  if (managementApiQueries.map(q => q.name.value).includes(definitionName)) {
    return ["queries", 'management-apis']
  }
  if (managementApiMutations.map(m => m.name.value).includes(definitionName)) {
    return ["mutations", 'management-apis']
  }

  return [null, null]
}

/**
 * @param {Array<{name: string, type: string, description: string, fields: Array<{name: string, type: string, description: string, isDeprecated: boolean}>}} definitions 
 * @param {string} definitionName 
 * @returns {string | null}
 */
export function getUiKitDefinitionPath(definitions, definitionName) {
  if (!definitions) {
    return null
  }

  const definitionPath = definitions.find(d => d.name === definitionName)?.path

  if (definitionPath == null) {
    return null
  }

  return definitionPath.split("/").pop().replace(".types.d.ts", "")
}

/**
 * 
 * @param {string} namedType 
 * @param {Array<{name: string, type: string, description: string, fields: Array<{name: string, type: string, description: string, isDeprecated: boolean}>}>} uiKitDefinitions 
 * @returns {string}
 */
export function getNamedTypePath(namedType, uiKitDefinitions) {
  const [apiDefinitionType, api] = getDefinitionType(namedType)

  if (apiDefinitionType != null) {
    return `/snippets/${api}/${apiDefinitionType}/${pascalToSlug(namedType)}.mdx`
  }

  const uiKitDefinitionType = getUiKitDefinitionPath(uiKitDefinitions, namedType)

  if (uiKitDefinitionType != null) {
    return `/snippets/ui-kit/${uiKitDefinitionType}/${namedType}.mdx`
  }

  return null
}

/**
 * @param {string} name 
 * @returns {string}
 */
export function pascalToSlug(name) {
  return name.replace(/([A-Z])/g, '-$1').toLowerCase().replace(/^-/, '');
}

/**
 * @param {string} name 
 * @returns {string}
 */
export function slugToPascal(name) {
  return name.charAt(0).toUpperCase() + name.slice(1).replace(/-([a-z])/g, (match, letter) => letter.toUpperCase())
}

export function slugToCamel(name) {
  return name.charAt(0).toLowerCase() + name.slice(1).replace(/-([a-z])/g, (match, letter) => letter.toUpperCase())
}

import { promises as fs } from "fs";
import { Spinner } from "@topcli/spinner";

export default async function main(pagesFiles) {
  const spinner = new Spinner().start("Generating API pages");

  const buildFrontMatter = (title, description, icon) => `---
title: "${title}"
description: "${description}"
icon: "${icon}"
---\n`;

  /**
   *
   * @param {'Management API' | 'Query API'} api
   * @returns
   */
  const queriesFrontMatter = (api) =>
    buildFrontMatter("Queries", `Available queries in the ${api}`, "magnifying-glass");

  const inputsFrontMatter = (api) =>
    buildFrontMatter("Inputs", `Input types in the ${api}`, "arrow-right-to-bracket");

  const enumsFrontMatter = (api) =>
    buildFrontMatter("Enums", `Enumeration types in the ${api}`, "list");

  const mutationsFrontMatter = (api) =>
    buildFrontMatter("Mutations", `Available mutations in the ${api}`, "pen-to-square");

  const unionsFrontMatter = (api) =>
    buildFrontMatter("Unions", `Union types in the ${api}`, "code-merge");

  const typesFrontMatter = (api) =>
    buildFrontMatter("Types", `Object types in the ${api}`, "cube");

  const scalarsFrontMatter = (api) =>
    buildFrontMatter("Scalars", `Scalar types in the ${api}`, "shapes");

  generatePage("queries", queriesFrontMatter("Query API"), "query-apis");
  generatePage("inputs", inputsFrontMatter("Query API"), "query-apis");
  generatePage("enums", enumsFrontMatter("Query API"), "query-apis");
  generatePage("mutations", mutationsFrontMatter("Query API"), "query-apis");
  generatePage("unions", unionsFrontMatter("Query API"), "query-apis");
  generatePage("types", typesFrontMatter("Query API"), "query-apis");
  generatePage("scalars", scalarsFrontMatter("Query API"), "query-apis");

  generatePage("queries", queriesFrontMatter("Management API"), "management-apis");
  generatePage("inputs", inputsFrontMatter("Management API"), "management-apis");
  generatePage("enums", enumsFrontMatter("Management API"), "management-apis");
  generatePage("mutations", mutationsFrontMatter("Management API"), "management-apis");
  generatePage("unions", unionsFrontMatter("Management API"), "management-apis");
  generatePage("types", typesFrontMatter("Management API"), "management-apis");
  generatePage("scalars", scalarsFrontMatter("Management API"), "management-apis");

  /**
   * @param {string} name
   * @param {string} frontMatter
   * @param {'query-apis' | 'management-apis'} api
   */
  async function generatePage(name, frontMatter, api) {
    let fileContent = "";

    const resultSubPath = {
      "query-apis": "docs/query-apis/api-reference",
      "management-apis": "docs/management-api",
    }[api];

    const resultPath = `${resultSubPath}/${name}.mdx`;

    fileContent = pagesFiles[`${api}/${name}`];

    fileContent = `${frontMatter}\n${fileContent}`;

    await fs.writeFile(resultPath, fileContent);
  }

  spinner.succeed("API pages generated successfully");
}


import { promises as fs } from 'fs'

const pathsToDelete = [
  'docs/management-api',
  'docs/query-apis/api-reference',
  'snippets/ui-kit'
]

for (const path of pathsToDelete) {
  const dir = await fs.readdir(path)
  for (const file of dir) {
    if (file.includes('overview')) continue
    await fs.rm(`${path}/${file}`, { recursive: true, force: true });
  }
}


import generateQueryApiSnippets from "./generate-query-api-snippets.js";
import generateManagementApiPages from "./generate-management-api-pages.js";
import patchEmbedSnippets from "./patch-embed-snippets.js";
import generateUiKitPropsSnippets from "./generate-ui-kit-props-snippets.js";

const { pagesFiles, patchExpandables } =
  await generateQueryApiSnippets();

await generateManagementApiPages(pagesFiles);

await patchEmbedSnippets(pagesFiles);

await generateUiKitPropsSnippets(patchExpandables);


import { promises as fs } from "fs";

export default async function main(pagesFiles) {
  const dirsToPatch = ["docs/query-apis"];

  for (const dir of dirsToPatch) {
    await patch(dir);
  }

  async function patch(dir) {
    const fullDir = await fs.readdir(dir, { recursive: true });
    for (const file of fullDir) {
      if (!file.endsWith(".mdx")) {
        await patch(`${dir}/${file}`);
        continue;
      }

      const content = await fs.readFile(`${dir}/${file}`, "utf8");
      const patchedContent = patchFile(content);

      if (patchedContent != null) {
        await fs.writeFile(`${dir}/${file}`, patchedContent);
      }
    }
  }

  /**
   * @param {string} fileContent
   */
  function patchFile(fileContent) {
    const splitContent = fileContent.split("\n");
    let embedSnipetLineIndex = splitContent.findIndex((line) =>
      line.includes("<EmbedSnippet")
    );
    if (embedSnipetLineIndex === -1) {
      const snippetForLineIndex = splitContent.findIndex((line) =>
        line.includes("{/*-- ------------------------ SNIPPET FOR")
      );
      const snippetForLineEndIndex = splitContent
        .slice(snippetForLineIndex + 1)
        .findIndex((line) =>
          line.includes("{/*-- ------------------------ SNIPPET FOR")
        );
      const path = splitContent[snippetForLineIndex]?.split(" ")?.[4];

      if (snippetForLineIndex !== -1 && path != null) {
        splitContent.splice(
          snippetForLineIndex,
          snippetForLineEndIndex + 2,
          `<EmbedSnippet path="${path}" />`
        );
        embedSnipetLineIndex = splitContent.findIndex((line) =>
          line.includes("<EmbedSnippet")
        );
      } else {
        return null;
      }
    }

    const embedSnippetLine = splitContent[embedSnipetLineIndex];
    const pathProp = embedSnippetLine.split(" ")[1];
    const path = pathProp.replace("path=", "").replaceAll('"', "");

    const snippet = pagesFiles[path];

    if (snippet == null) {
      throw new Error(`Snippet not found for ${path}`);
    }

    splitContent[
      embedSnipetLineIndex
    ] = `{/*-- ------------------------ SNIPPET FOR ${path} ------------------------ */}\n${snippet}\n{/*-- ------------------------ SNIPPET FOR ${path} ------------------------ */}`;

    const patchedFile = splitContent.join("\n");

    return patchedFile;
  }
}


import * as ts from "typescript";
import fs from "fs";

export default function getRadixTypeDefinitionAsJson(typeName) {
  const filePaths = fs
    .readdirSync("node_modules/@radix-ui/themes/src", { recursive: true })
    .map((file) => `node_modules/@radix-ui/themes/src/${file}`)
    .filter((file) => file.endsWith(".ts") || file.endsWith(".tsx"));

  const fileName = `node_modules/@radix-ui/themes/src/components/${typeName.replace("Props", ".tsx")}`;

  // Create a program to analyze the source file
  const program = ts.createProgram(filePaths, {});
  const checker = program.getTypeChecker();
  // Get the source file
  const sourceFile = program.getSourceFile(fileName);
  if (!sourceFile) {
    throw new Error(`File not found: ${fileName}`);
  }
  let targetType;
  ts.forEachChild(sourceFile, (node) => {
    if (ts.isInterfaceDeclaration(node) || ts.isTypeAliasDeclaration(node)) {
      const symbol = checker.getSymbolAtLocation(node.name);
      if (symbol && symbol.name === typeName) {
        targetType = checker.getDeclaredTypeOfSymbol(symbol);
      }
    }
  });
  if (!targetType) {
    throw new Error(`Type not found: ${typeName}`);
  }
  function getTypeLiteralText(type) {
    return checker.typeToString(type);
  }
  function serializeTypeLiteralText(type) {
    const result = {};
    // Get properties of the type
    const members = checker.getPropertiesOfType(type);
    for (const member of members) {
      const memberDeclaration = member.valueDeclaration || member.declarations?.[0];
      if (memberDeclaration) {
        const memberType = checker.getTypeOfSymbolAtLocation(member, memberDeclaration);
        // Get the literal text of the member's type
        result[member.name] = getTypeLiteralText(memberType);
      } else {
        result[member.name] = "unknown";
      }
    }
    return result;
  }
  return serializeTypeLiteralText(targetType);
}


Architecture

Features

Schemaless ingestion

Message deduplication

How does the Kafka Data Pool work?

Supported formats

Transforming data

Overview

Kafka to ClickHouse

Propel docs

Serverless Cloud

Ingestion

Query APIs

SQL Reference

Management API

REST API Reference

API Reference

Embeddable UI

Changelog

Console

Sign up

Guides, resources, and references to build with Propel.

Propel documentation

Learn why ClickHouse is **so fast** for analytical use cases.

What is ClickHouse®?

Built for developers by ex-Twilio engineers.

Why Propel?

Learn about Propel's cloud-native ClickHouse® architecture.

Get up and running with Propel in under 5 minutes.

Quickstart

Connect using ClickHouse or PostgreSQL clients and SDKs.

Connect & SDKs

Create Data Pools via the API or Terraform.

Creating tables

Individual and bulk inserts into ClickHouse

Inserting data

Real-time and batch updates in ClickHouse.

Updating data

Real-time and batch deletes in ClickHouse.

Deleting data

Changing schemas

Materialized Views

Query data from other databases and data warehouses.

Federation

Applications

Fine-grained control over Data Pool access with column and row-level rules

Access Policies

Using JWT tokens to secure multi-tenant applications.

Multi-tenant JWT tokens

Guide to selecting the optimal Propeller size for your use case

Choosing a Propeller

How to select and use the optimal default timestamp for your Data Pool

Default Timestamp

Default timestamp

How to select the best table engine and sorting key in ClickHouse

Table Engine & Sorting Key

Selecting table engine and sorting key

Example section for showcasing API endpoints

Partition Key

Partition key

Propel Terraform provider

Advanced Terraform examples for Propel resources

Advanced Examples

Advanced examples

Configure Single Sign-On (SSO) for user authentication.

Single Sign-On (SSO)

Starter Apps

Authentication

Layout

Querying Metrics

Logging

Customize loading, error and empty states.

Default States

Customizing default states

Chart.js configuration

Theme overview

Understand the color system and its application in your theme.

Color

Dark mode

Typography

Spacing

Define responsive breakpoints for your layout.

Breakpoints

Choose the border radius for UI elements.

Radius

Select shadow styles for your components.

Shadows

Customize cursor styles for interactive elements.

Cursors

Time Series

Counter

Data Grid

Leaderboard

Pie Chart

Time Range Picker

Simple Filter

Choose a time granularity for aggregation.

Time Grain Picker

Group By

Access Token Provider

AccessToken Provider

Filter Provider

Filter provider

Log Provider

Log provider

Theme Provider

Theme provider

useSql

A React Hook for fetching a  value from the Counter API.

useCounter

A React Hook for fetching tabular data from the Data Grid API.

useDataGrid

A React Hook for fetching ranked data from the Leaderboard API.

useLeaderboard

A React Hook for ready-to-chart time series data from the Time Series API.

useTimeSeries

A React Hook for fetching unique values from the Top Values API.

useTopValues

2024

2024 Changelog

2023

2023 Changelog

2022

2022 Changelog

Execute SQL queries from your application.

SQL API

Get a single aggregated value for a time range.

Get aggregated values grouped by common dimensions.

Get multiple aggregated values grouped by common dimensions.

Metric Report

Metric report

Get the most frequent non-null values in a column.

Top Values

Top values

Metrics

Learn about the different types of metrics.

Types of Metrics

Define Metrics using the console, API, or Terraform.

Defining Metrics

Querying JSON

Querying JSON data

Multiple queries

Make multiple GraphQL queries in a single request

Timezones

Working with timezones

Pagination

Unique names

Learn the key elements of ClickHouse SQL syntax.

Syntax

Data Types

Data types

Table Engines

Table engines

Operators

SELECT

TABLES

SHOW TABLES

Management API reference

Queries

Available mutations in the Management API

Mutations

Types

Inputs

Enums

Unions

Scalars

Query API reference

Setup guide

Kafka setup guide

Webhooks to ClickHouse

Setup Guide

Webhooks setup guide

Ingest data from Amazon Data Firehose into Propel.

Amazon Data Firehose to ClickHouse

Ingesting data from Amazon Data Firehose streams to Propel

Amazon Data Firehose setup guide

Segment to ClickHouse

Ingesting events from Twilio Segment to Propel

Twilio Segment setup guide

Snowflake to ClickHouse

Ingesting data from Snowflake into Propel.

Snowflake setup guide

Amazon S3 to ClickHouse

Ingesting data from Amazon S3 into Propel.

Amazon S3 setup guide

Ingest data from Amazon DynamoDB into Propel.

Amazon DynamoDB to ClickHouse

Ingesting data from Amazon DynamoDB to Propel

Feature name	Supported	Notes
Real-time ingestion	✅	See How the Kafka Data Pool works.
Deduplication	✅	See the deduplication section.
Batch Delete API	✅	See Batch Delete API.
Batch Update API	✅	See Batch Update API.
API configurable	✅	See API](/docs/management-api) docs.
Terraform configurable	✅	See Terraform docs.

Column	Type	Description
`_timestamp`	TIMESTAMP	The timestamp of the message.
`_topic`	STRING	The Kafka topic
`_key`	STRING	The key of the message.
`_offset`	INT64	The offset of the message.
`_partition`	INT64	The partition of Kafka topic.
`_propel_payload`	JSON	The raw message Payload in JSON.
`_propel_received_at`	TIMESTAMP	When the message is read by Propel.

Get Started

Streaming

Data warehouses

Databases

ETL Platforms

Kafka to ClickHouse

Get started with Kafka

Architecture

Features

How does the Kafka Data Pool work?

Schemaless ingestion

Message deduplication

Supported formats

Transforming data

Get Started

Streaming

Data warehouses

Databases

ETL Platforms

Get started with Kafka

​Architecture

​Features

​How does the Kafka Data Pool work?

​Schemaless ingestion

​Message deduplication

​Supported formats

​Transforming data

Architecture

Features

How does the Kafka Data Pool work?

Schemaless ingestion

Message deduplication

Supported formats

Transforming data