upload.js
For more information on using this script, see here
/** The API Gateway URL */
const API_URLS = {
environment1: "https://abcdefghij.execute-api.us-east-1.amazonaws.com/api",
environment2: "https://klmnopqrst.execute-api.us-east-1.amazonaws.com/api"
};
/** The OAuth issuer URL */
const OAUTH_URLS = {
environment1: "https://{url}/idp",
environment2: "https://{url}/idp"
};
/** The OAuth scope(s) to be requested */
const CLIENT_SCOPE = "environment_authorization";
// *****************************************************************************************************************************
// The following code is a Node.js script that uploads files to S3 using presigned URLs.
// *****************************************************************************************************************************
const fs = require('fs');
const http = require('http');
const https = require('https');
const path = require('path');
const url = require('url');
if (process.argv.length < 3) {
help();
return;
}
const env = process.argv[2];
if (!["environment1", "environment2"].includes(env)) {
log(`Environment ${env} not found!`);
help();
return;
}
const inputpath = process.argv[3];
if (!fs.existsSync(inputpath)) {
log(`${inputpath} not found!`);
help();
return;
}
const outputpath = process.argv[4];
if (outputpath && !fs.existsSync(outputpath)) {
try {
fs.mkdirSync(outputpath);
} catch (error) {
log(`Failed to create output folder ${outputpath}!`);
help();
return;
}
}
const API_URL = API_URLS[env];
if (!API_URL || API_URL === "") {
log(`Missing API_URL for ${env} environment!`);
return;
}
const OAUTH_URL = OAUTH_URLS[env];
if (!OAUTH_URL || OAUTH_URL === "") {
log(`Missing OAUTH_URL for ${env} environment!`);
return;
}
const credentials = getCredentials(env);
if (!credentials.client_id || !credentials.client_secret) {
log(`Missing client_id or client_secret in credentials.json for ${env} environment!`);
return;
}
const totalStart = performance.now();
const files = fs.lstatSync(inputpath).isDirectory()
? fs.readdirSync(inputpath).map(file => path.join(inputpath, file))
: [inputpath];
log(`Uploading ${files.length} files to S3`);
log("Authorizing...", true);
login()
.then(async token => {
log("Logged in");
const uploadStart = performance.now();
const getUrls = new Map();
for (file of files) {
const filename = path.basename(file);
log(`Uploading ${filename} to S3 with presigned URL`, true);
const getUrl = await upload(file, token);
if (getUrl) {
getUrls.set(filename, getUrl);
}
}
const uploadTime = ellapsedTime(uploadStart);
log(`${getUrls.size} of ${files.length} uploads complete in ${uploadTime}`);
if (outputpath) {
const downloadStart = performance.now();
const delay = seconds => new Promise(resolve => setTimeout(resolve, seconds * 1000));
let pendingFiles = Array.from(getUrls.keys());
while (pendingFiles.length > 0) {
const message = `${files.length - pendingFiles.length} of ${files.length} downloads complete`;
let progress = '';
// Waiting 20 seconds
for (let i = 0; i < 10; i++) {
progress += '.';
log(`${message}${progress}`, true);
await delay(2);
}
for (file of pendingFiles) {
const filename = path.basename(file, path.extname(file));
const outputFileType = `${filename}.json`;
const outputFilename = path.join(outputpath, outputFileType);
const getUrl = getUrls.get(file);
if (getUrl && await download(getUrl, outputFilename)) {
log(`Downloaded ${outputFileType} from S3 with presigned URL`, true);
getUrls.delete(file);
}
}
pendingFiles = Array.from(getUrls.keys());
}
const downloadTime = ellapsedTime(downloadStart);
log(`${getUrls.size - pendingFiles.length} of ${getUrls.size} downloads complete in ${downloadTime}`);
console.timeEnd("Download time");
}
const totalTime = ellapsedTime(totalStart);
log(`Total time: ${totalTime}`);
process.exit();
})
.catch(error => {
console.error(error.message);
process.exit();
});
/**
* Displays the help message for the script.
*/
function help() {
log("Usage: node upload.js <env> <input> <output>");
log(" env - The name of the environment to upload files to (environment1, environment2, etc)");
log(" input - The path to a file or folder to upload to S3");
log(" output - (Optional) The path to a folder to download files from S3");
}
/**
* Gets the credentials for the specified environment.
* @param {string} env
* @returns {OAuthCredentials} The OAuth credentials
*/
function getCredentials(env) {
const filename = path.join(__dirname, 'credentials.json');
const credentials = JSON.parse(fs.readFileSync(filename, 'utf8'));
return credentials[env];
}
/**
* Logs the user in to the OAuth server.
* @returns {Promise<string>} The access token
*/
async function login() {
const accessToken = await getAccessToken();
if (accessToken) {
return accessToken;
}
throw new Error("Authentication failure");
}
/**
* Gets the access token from the OAuth server.
* @returns {Promise<string>} The access token
*/
async function getAccessToken() {
const params = new url.URLSearchParams({
grant_type: "urn:hyland:params:oauth:grant-type:api-credentials",
client_id: credentials.client_id,
client_secret: credentials.client_secret,
scope: CLIENT_SCOPE
}).toString();
const response = await postData(params, `${OAUTH_URL}/connect/token`, { 'Content-Type': 'application/x-www-form-urlencoded' });
const responseJson = JSON.parse(response);
return responseJson.access_token;
}
/**
* Uploads a file to S3 and returns the presigned GET URL.
* @param {string} file Path to the file to upload
* @param {string} token Access Token
* @returns {Promise<string>} The Presigned S3 GET URL
*/
async function upload(file, token) {
const length = fs.statSync(file).size;
const filename = path.basename(file);
const apiPresignUrl = `${API_URL}/presign`;
const presignHeaders = {
'Authorization': `Bearer ${token}`
};
const options = {
'normalization': {
'quotations': true
},
"chunking": true,
"embedding": true
}
const response = await postJson(apiPresignUrl, presignHeaders, JSON.stringify(options));
const responseJson = JSON.parse(response);
if (responseJson.put_url) {
log(`${filename} - ${responseJson.job_id}`);
putUrl = responseJson.put_url;
const putHeaders = {
'Content-Type': 'application/octet-stream',
'Content-Length': length
};
await putFile(file, putUrl, putHeaders);
} else {
log(`${filename} - Error ${response}`);
}
return responseJson.get_url;
}
/**
* Downloads a file from S3 using a presigned URL.
* @param {string} url
* @param {string} file Path to the file to download
* @returns
*/
async function download(url, file) {
let response;
try {
response = await getBinary(url, {});
} catch {
response = undefined;
}
if (response === undefined) {
return false;
}
const fileBuffer = Buffer.from(response, 'base64');
const text = fileBuffer.toString('ascii');
if (text.includes("<Error><Code>NoSuchKey</Code><Message>The specified key does not exist.</Message>")) {
return false;
}
fs.writeFileSync(file, fileBuffer);
return true;
}
/**
* Sends a POST request to the specified URL.
* @param {string} url
* @param {http.OutgoingHttpHeaders} headers HTTP request headers
* @returns {Promise<string>} The response body
*/
function postJson(url, headers, body) {
return new Promise((resolve, reject) => {
const callback = response => resolve(Buffer.from(response, 'base64').toString('ascii'));
const request = httpsRequest('POST', url, headers, callback, reject);
request.write(body);
request.end();
});
}
/**
* Sends a GET request to the specified URL.
* @param {string} url
* @param {http.OutgoingHttpHeaders} headers HTTP request headers
* @returns {Promise<string>} The response body
*/
function getBinary(url, headers) {
return new Promise((resolve, reject) => {
const request = httpsRequest('GET', url, headers, resolve, reject);
request.end();
});
}
/**
* Sends a file as a PUT request to the specified URL.
* @param {string} file Path to the file to upload
* @param {string} url URL to upload the file to
* @param {http.OutgoingHttpHeaders} headers HTTP request headers
* @returns {Promise<string>} The response body
*/
function putFile(file, url, headers) {
return new Promise((resolve, reject) => {
const callback = response => resolve(Buffer.from(response, 'base64').toString('ascii'));
const request = httpsRequest('PUT', url, headers, callback, reject);
fs.createReadStream(file).pipe(request);
});
}
/**
* Sends a POST request to the specified URL.
* @param {string} body HTTP request body
* @param {string} url URL to send the POST request to
* @param {http.OutgoingHttpHeaders} headers HTTP request headers
* @returns {Promise<string>} The response body
*/
function postData(body, url, headers) {
return new Promise((resolve, reject) => {
const callback = response => resolve(Buffer.from(response, 'base64').toString('ascii'));
const request = httpsRequest('POST', url, headers, callback, reject);
request.write(body);
request.end();
});
}
/**
* Sends an HTTP request to the specified URL.
* @param {string} method
* @param {string} url
* @param {http.OutgoingHttpHeaders} headers
* @param {function(string):void} endCallback
* @param {function(Error):void} errorCallback
* @returns {http.ClientRequest} HTTP request object
*/
function httpsRequest(method, url, headers, endCallback, errorCallback) {
const options = httpRequestOptions(method, url, headers);
const data = [];
return https.request(options, (response) => {
response.on('data', (chunk) => {
data.push(chunk);
});
response.on('end', () => {
const buffer = Buffer.concat(data);
const body = buffer.toString('base64');
endCallback(body);
});
response.on('error', (error) => {
errorCallback(error);
});
});
}
/**
* Builds the options for an HTTP request.
* @param {string} method HTTP method
* @param {string} url URL to send the request to
* @param {http.OutgoingHttpHeaders} headers HTTP request headers
* @returns {https.RequestOptions} HTTP request options
*/
function httpRequestOptions(method, url, headers) {
const requestUrl = new URL(url);
return {
hostname: requestUrl.hostname,
port: 443,
path: requestUrl.pathname + requestUrl.search,
method: method,
headers: headers
};
}
/**
* Logs a message to the console.
* @param {string} message
* @param {boolean} overwriteThisLine
*/
function log(message, overwriteThisLine = false) {
process.stdout.cursorTo(0);
process.stdout.write(message);
process.stdout.clearLine(1);
if (!overwriteThisLine) {
process.stdout.write("\n");
}
}
/**
* Returns the ellapsed time since the specified start time.
* @param {number} startTime
* @returns {string} The ellapsed time in seconds
*/
function ellapsedTime(startTime) {
const difference = performance.now() - startTime;
const ellapsedSeconds = Math.round((difference + Number.EPSILON) / 100) / 10; // round to 1 decimal place
return `${ellapsedSeconds} seconds`;
}
/**
* The OAuth credentials.
*/
class OAuthCredentials {
/** The OAuth client id */
client_id;
/** The OAuth client secret */
client_secret;
}