Skip to main content

JavaScript Sample

A Node.js script covering the full Data Curation API workflow: authenticate → upload files → poll status → download results.

Download the JavaScript script (.js)

Setup​

Options​

const DC_API_URL = "https://knowledge-enrichment.ai.app.hyland.com/latest/api/data-curation";
const DC_OPTIONS = {
  normalization: { quotations: true, dashes: true }, // Normalize quotation marks and dashes
  chunking: true,      // Enable semantic chunking
  chunk_size: 1000,    // Target chunk size in characters
  embedding: false,    // Generate embeddings for chunks (requires chunking: true)
  json_schema: false,  // Output format: false, "FULL", "MDAST", or "PIPELINE"
  pii: false           // false, "detection", "redaction", or { mode, entity_redaction }
};

Credentials​

// Configure an external application in the admin console:
// https://admin.app.hyland.com/external-systems/external-applications
// The application must have the environment_authorization scope and a Data Curation API subscription.
const OAUTH_URL = "https://auth.app.hyland.com/idp";
const CLIENT_ID = "YOUR_CLIENT_ID";
const CLIENT_SECRET = "YOUR_CLIENT_SECRET";

Helper Functions​

const https = require("https");
const fs = require("fs");
const path = require("path");
const url = require("url");

function request(method, targetUrl, headers, body) {
  return new Promise((resolve, reject) => {
    const parsed = new url.URL(targetUrl);
    const opts = {
      method,
      hostname: parsed.hostname,
      path: parsed.pathname + parsed.search,
      headers
    };
    const req = https.request(opts, (res) => {
      const chunks = [];
      res.on("data", (c) => chunks.push(c));
      res.on("end", () => resolve(Buffer.concat(chunks)));
    });
    req.on("error", reject);
    if (body) req.write(body);
    req.end();
  });
}

Workflow​

List Files to Upload​

const filePaths = ["input/document.pdf"];
console.log(`Uploading ${filePaths.length} file(s)`);
Sample Output
Uploading 1 file(s)

Get Access Token​

async function getAccessToken() {
  const credentials = Buffer.from(`${CLIENT_ID}:${CLIENT_SECRET}`).toString("base64");
  const body = new url.URLSearchParams({
    grant_type: "client_credentials",
    scope: "environment_authorization"
  }).toString();
  const response = await request(
    "POST",
    `${OAUTH_URL}/connect/token`,
    {
      "Content-Type": "application/x-www-form-urlencoded",
      Authorization: `Basic ${credentials}`
    },
    body
  );
  return JSON.parse(response.toString()).access_token;
}

const token = await getAccessToken();
console.log("Access token obtained");
Sample Output
Access token obtained

Get Presign URL​

async function getPresignUrl(token) {
  const response = await request(
    "POST",
    `${DC_API_URL}/presign`,
    {
      Authorization: `Bearer ${token}`,
      "Content-Type": "application/json"
    },
    JSON.stringify(DC_OPTIONS)
  );
  return JSON.parse(response.toString());
}

const presignResults = [];
for (const filePath of filePaths) {
  const { put_url, get_url, job_id } = await getPresignUrl(token);
  presignResults.push({ filePath, put_url, get_url, job_id });
  console.log(`${filePath} - job_id: ${job_id}`);
}
Sample Output
input/document.pdf - job_id: API_8d5b8381-567d-4471-b0ee-619b5b7601c7

Upload Files​

async function uploadFile(filePath, putUrl) {
  const data = fs.readFileSync(filePath);
  await request(
    "PUT",
    putUrl,
    {
      "Content-Type": "application/octet-stream",
      "Content-Length": data.length
    },
    data
  );
}

for (const { filePath, put_url } of presignResults) {
  await uploadFile(filePath, put_url);
  console.log(`Uploaded: ${filePath}`);
}
console.log(`${filePaths.length} of ${filePaths.length} uploads complete`);
Sample Output
Uploaded: input/document.pdf
1 of 1 uploads complete

Get Status​

async function getStatus(getUrl) {
  const response = await request("GET", getUrl, {});
  const body = response.toString();
  // Before the result is ready, S3 returns a NoSuchKey XML error
  if (body.includes("<Error>")) return null;
  return JSON.parse(body);
}

async function pollAllResults(presignResults, intervalMs = 5000, timeoutMs = 300000) {
  const results = new Map();
  const pending = [...presignResults];
  let elapsed = 0;
  while (pending.length > 0 && elapsed < timeoutMs) {
    await new Promise((r) => setTimeout(r, intervalMs));
    elapsed += intervalMs;
    for (let i = pending.length - 1; i >= 0; i--) {
      const { filePath, get_url } = pending[i];
      const result = await getStatus(get_url);
      if (result) {
        results.set(filePath, result);
        pending.splice(i, 1);
        console.log(`Done: ${filePath}`);
      }
    }
    if (pending.length > 0) console.log(`Waiting... ${pending.length} file(s) remaining`);
  }
  if (pending.length > 0) throw new Error("Timed out waiting for results");
  return results;
}

const outputResultsMap = await pollAllResults(presignResults);
const outputResultsJson = [...outputResultsMap.values()];
console.log(`${outputResultsJson.length} result(s) ready`);
Sample Output
Waiting... 1 file(s) remaining
Done: input/document.pdf
1 result(s) ready

Download Results​

const OUTPUT_DIR = "output";
if (!fs.existsSync(OUTPUT_DIR)) fs.mkdirSync(OUTPUT_DIR);

for (const [filePath, result] of outputResultsMap) {
  const filename = path.basename(filePath, path.extname(filePath));
  const outputPath = path.join(OUTPUT_DIR, `${filename}.json`);
  fs.writeFileSync(outputPath, JSON.stringify(result, null, 2));
  console.log(`Saved: ${outputPath}`);
}
console.log(`${outputResultsJson.length} of ${outputResultsJson.length} downloads complete`);
Sample Output
Saved: output/document.json
1 of 1 downloads complete

Parse Results​

const outputResults = outputResultsJson.map((result) => {
  const chunksJson = result.markdown?.chunks ?? [];
  const locationsJson = result.markdown?.locations ?? [];
  const chunks = chunksJson.map((text, i) => ({
    text,
    location: locationsJson[i] ?? ""
  }));
  return {
    markdownOutput: result.markdown?.output ?? "",
    chunks
  };
});

outputResults.forEach((parsed, i) => {
  console.log(`Result ${i}: ${parsed.chunks.length} chunks`);
  parsed.chunks.forEach((chunk, j) => {
    const text = chunk.text.length > 80 ? chunk.text.slice(0, 80) : chunk.text;
    console.log(`  [${j}] ${text.trim()}`);
  });
});
Sample Output
Result 0: 6 chunks
  [0] # Test Document
  [1] ## Header 2

 Test paragraph.
  [2] ### Header 3

 Another paragraph.
  [3] #### Header 4

 Even more paragraph.
  [4] ## Bullets

 * Point 1 * Point 2 * Point 3
  [5] ## Table

Display Markdown​

outputResults.forEach((parsed) => {
  console.log("--- Markdown Output ---");
  console.log(parsed.markdownOutput);
  console.log("-----------------------");
});
Sample Output
--- Markdown Output ---
# Test Document

## Header 2

Test paragraph.

### Header 3

Another paragraph.

#### Header 4

Even more paragraph.

## Bullets

* Point 1
* Point 2
* Point 3
  * Sub-point 3.1
  * Sub-point 3.2

## Table

| **Name** | **Value** | **Notes**       |
| -------- | --------- | --------------- |
| Text     | ABC       | Text Value      |
| Number   | 123       | Numerical Value |
| Symbol   | 🤣🤣🤣    | Emoji symbols   |
-----------------------

Display JSON​

outputResultsJson.forEach((result) => {
  console.log(JSON.stringify(result, null, 2));
});
Sample Output
{
  "markdown": {
    "output": "<!-- LOC: 1, (96,120,720,151) -->\n# Test Document\n...",
    "locations": [
      "<!-- LOC: 1, (96,164,720,189) -->",
      "<!-- LOC: 1, (96,196,720,214) -->",
      "..."
    ],
    "chunks": [
      "# Test Document\n\n",
      "## Header 2\n\n Test paragraph.",
      "..."
    ]
  }
}