Nasdaq new components

components/nasdaq_data_link_time_series_and_table_data_/actions/export-table/export-table.mjs

@@ -0,0 +1,91 @@
+import app from "../../nasdaq_data_link_time_series_and_table_data_.app.mjs";
+import utils from "../../common/utils.mjs";
+
+export default {
+  key: "nasdaq_data_link_time_series_and_table_data_-export-table",
+  name: "Export Table (Bulk Download)",
+  description: "Exports an entire table or a filtered subset as a zipped CSV file. Returns a download link for the data. Premium subscribers can use this feature up to 60 times per hour. [See the documentation](https://docs.data.nasdaq.com/docs/large-table-download)",
+  version: "0.0.1",
+  type: "action",
+  props: {
+    app,
+    publisher: {
+      propDefinition: [
+        app,
+        "publisher",
+      ],
+    },
+    table: {
+      propDefinition: [
+        app,
+        "table",
+      ],
+    },
+    columns: {
+      propDefinition: [
+        app,
+        "columns",
+        ({
+          publisher, table,
+        }) => ({
+          publisher,
+          table,
+        }),
+      ],
+    },
+    filters: {
+      type: "object",
+      label: "Row Filters",
+      description: "Filter rows based on column values. Use column names as keys and values to filter by. For example: `{ \"ticker\": \"SPY\", \"date\": \"2024-01-01\" }`. Only filterable columns can be used (check table metadata).",
+      optional: true,
+    },
+    filterOperators: {
+      type: "object",
+      label: "Filter Operators",
+      description: "Apply operators to filters. Format: `{ \"column.operator\": \"value\" }`. Available operators: `.gt` (greater than), `.lt` (less than), `.gte` (greater than or equal), `.lte` (less than or equal). Example: `{ \"date.gte\": \"2024-01-01\", \"date.lte\": \"2024-12-31\" }`",
+      optional: true,
+    },
+  },
+  async run({ $ }) {
+    const {
+      app,
+      publisher,
+      table,
+      columns,
+      filters,
+      filterOperators,
+    } = this;
+
+    const response = await app.tableData({
+      $,
+      publisher,
+      table,
+      params: {
+        "qopts.export": true,
+        ...utils.parseJson(filters),
+        ...utils.parseJson(filterOperators),
+        ...(Array.isArray(columns) && columns?.length
+          ? {
+            "qopts.columns": columns.join(","),
+          }
+          : undefined
+        ),
+      },
+    });
+
+    const status = response?.datatable_bulk_download?.file?.status;
+    const link = response?.datatable_bulk_download?.file?.link;
+
+    if (status === "fresh" && link) {
+      $.export("$summary", `Table ${publisher}/${table} is ready for download. The download link is valid for 30 minutes.`);
+
+    } else if (status === "creating" || status === "regenerating") {
+      $.export("$summary", `Export job for table ${publisher}/${table} is ${status}. Please retry in a few moments to get the download link.`);
+
+    } else {
+      $.export("$summary", `Export initiated for table ${publisher}/${table}`);
+    }
+
+    return response;
+  },
+};

components/nasdaq_data_link_time_series_and_table_data_/actions/get-table-data/get-table-data.mjs

@@ -0,0 +1,81 @@
+import app from "../../nasdaq_data_link_time_series_and_table_data_.app.mjs";
+import utils from "../../common/utils.mjs";
+
+export default {
+  key: "nasdaq_data_link_time_series_and_table_data_-get-table-data",
+  name: "Get Table Data",
+  description: "Retrieves data from a specific Nasdaq Data Link table with automatic pagination. Supports filtering by columns and rows. [See the documentation](https://docs.data.nasdaq.com/docs/tables-1)",
+  version: "0.0.1",
+  type: "action",
+  props: {
+    app,
+    publisher: {
+      propDefinition: [
+        app,
+        "publisher",
+      ],
+    },
+    table: {
+      propDefinition: [
+        app,
+        "table",
+      ],
+    },
+    columns: {
+      propDefinition: [
+        app,
+        "columns",
+        ({
+          publisher, table,
+        }) => ({
+          publisher,
+          table,
+        }),
+      ],
+    },
+    filters: {
+      type: "object",
+      label: "Row Filters",
+      description: "Filter rows based on column values. Use column names as keys and values to filter by. For example: `{ \"ticker\": \"SPY\", \"date\": \"2024-01-01\" }`. Only filterable columns can be used (check table metadata).",
+      optional: true,
+    },
+    filterOperators: {
+      type: "object",
+      label: "Filter Operators",
+      description: "Apply operators to filters. Format: `{ \"column.operator\": \"value\" }`. Available operators: `.gt` (greater than), `.lt` (less than), `.gte` (greater than or equal), `.lte` (less than or equal). Example: `{ \"date.gte\": \"2024-01-01\", \"date.lte\": \"2024-12-31\" }`",
+      optional: true,
+    },
+  },
+  async run({ $ }) {
+    const {
+      app,
+      publisher,
+      table,
+      columns,
+      filters,
+      filterOperators,
+    } = this;
+
+    const response = await app.paginate({
+      fn: app.tableData,
+      args: {
+        $,
+        publisher,
+        table,
+        params: {
+          ...utils.parseJson(filters),
+          ...utils.parseJson(filterOperators),
+          ...(Array.isArray(columns) && columns?.length
+            ? {
+              "qopts.columns": columns.join(","),
+            }
+            : undefined
+          ),
+        },
+      },
+    });
+
+    $.export("$summary", `Successfully retrieved ${response.length} records`);
+    return response;
+  },
+};

components/nasdaq_data_link_time_series_and_table_data_/actions/get-table-metadata/get-table-metadata.mjs

@@ -0,0 +1,40 @@
+import app from "../../nasdaq_data_link_time_series_and_table_data_.app.mjs";
+
+export default {
+  key: "nasdaq_data_link_time_series_and_table_data_-get-table-metadata",
+  name: "Get Table Metadata",
+  description: "Retrieves metadata for a specific Nasdaq Data Link table, including column names, types, filterable columns, and primary keys. [See the documentation](https://docs.data.nasdaq.com/docs/tables-1)",
+  version: "0.0.1",
+  type: "action",
+  props: {
+    app,
+    publisher: {
+      propDefinition: [
+        app,
+        "publisher",
+      ],
+    },
+    table: {
+      propDefinition: [
+        app,
+        "table",
+      ],
+    },
+  },
+  async run({ $ }) {
+    const {
+      app,
+      publisher,
+      table,
+    } = this;
+
+    const response = await app.tableMetadata({
+      $,
+      publisher,
+      table,
+    });
+
+    $.export("$summary", `Successfully retrieved metadata for table \`${publisher}/${table}\``);
+    return response;
+  },
+};

components/nasdaq_data_link_time_series_and_table_data_/common/utils.mjs

@@ -0,0 +1,44 @@
+const parseJson = (input, maxDepth = 100) => {
+  const seen = new WeakSet();
+  const parse = (value) => {
+    if (maxDepth <= 0) {
+      return value;
+    }
+    if (typeof(value) === "string") {
+      // Only parse if the string looks like a JSON object or array
+      const trimmed = value.trim();
+      if (
+        (trimmed.startsWith("{") && trimmed.endsWith("}")) ||
+        (trimmed.startsWith("[") && trimmed.endsWith("]"))
+      ) {
+        try {
+          return parseJson(JSON.parse(value), maxDepth - 1);
+        } catch (e) {
+          return value;
+        }
+      }
+      return value;
+    } else if (typeof(value) === "object" && value !== null && !Array.isArray(value)) {
+      if (seen.has(value)) {
+        return value;
+      }
+      seen.add(value);
+      return Object.entries(value)
+        .reduce((acc, [
+          key,
+          val,
+        ]) => Object.assign(acc, {
+          [key]: parse(val),
+        }), {});
+    } else if (Array.isArray(value)) {
+      return value.map((item) => parse(item));
+    }
+    return value;
+  };
+
+  return parse(input);
+};
+
+export default {
+  parseJson,
+};

components/nasdaq_data_link_time_series_and_table_data_/nasdaq_data_link_time_series_and_table_data_.app.mjs

@@ -1,11 +1,106 @@
+import { axios } from "@pipedream/platform";
+
 export default {
   type: "app",
   app: "nasdaq_data_link_time_series_and_table_data_",
-  propDefinitions: {},
+  propDefinitions: {
+    publisher: {
+      type: "string",
+      label: "Publisher Code",
+      description: "The publisher code (e.g., `MER`, `ETFG`, `AR`, `NDAQ`). This is the first part of the datatable code. If the code is `MER/F1`, then `MER` is the publisher code and `F1` is the table code.",
+    },
+    table: {
+      type: "string",
+      label: "Table Code",
+      description: "The table code (e.g., `F1`, `FUND`, `MWCS`, `RTAT10`). This is the second part of the datatable code. If the code is `MER/F1`, then `F1` is the table code.",
+    },
+    columns: {
+      type: "string[]",
+      label: "Columns",
+      description: "Request data from specific columns. If you want to query for multiple columns, include the column names as array items",
+      optional: true,
+      async options({
+        publisher, table,
+      }) {
+        if (!publisher || !table) {
+          return [];
+        }
+        const { datatable: { columns } } = await this.tableMetadata({
+          publisher,
+          table,
+        });
+        return columns.map(({ name }) => name);
+      },
+    },
+  },
   methods: {
-    // this.$auth contains connected account data
-    authKeys() {
-      console.log(Object.keys(this.$auth));
+    getUrl(path) {
+      return `https://data.nasdaq.com/api/v3${path}`;
+    },
+    getHeaders(headers) {
+      return {
+        ...headers,
+        "Accept": "application/json",
+        "X-Api-Token": this.$auth.api_key,
+      };
+    },
+    makeRequest({
+      $ = this, path, headers, ...args
+    } = {}) {
+      return axios($, {
+        ...args,
+        url: this.getUrl(path),
+        headers: this.getHeaders(headers),
+      });
+    },
+    tableMetadata({
+      publisher, table, ...args
+    }) {
+      return this.makeRequest({
+        path: `/datatables/${publisher}/${table}/metadata.json`,
+        ...args,
+      });
+    },
+    tableData({
+      publisher, table, ...args
+    }) {
+      return this.makeRequest({
+        path: `/datatables/${publisher}/${table}.json`,
+        ...args,
+      });
+    },
+    async paginate({
+      fn, args = {}, maxRequests = 3,
+    } = {}) {
+      let allData = [];
+      let cursorId = null;
+      let requestCount = 0;
+      let hasMorePages = true;
+
+      while (hasMorePages && requestCount < maxRequests) {
+        const response = await fn({
+          ...args,
+          params: {
+            ...args.params,
+            "qopts.per_page": 100,
+            ...(cursorId
+              ? {
+                "qopts.cursor_id": cursorId,
+              }
+              : undefined
+            ),
+          },
+        });
+
+        const pageData = response?.datatable?.data || [];
+        allData = allData.concat(pageData);
+
+        cursorId = response?.meta?.next_cursor_id;
+        hasMorePages = !!cursorId;
+        requestCount++;
+      }
+
+      return allData;
     },
   },
 };

components/nasdaq_data_link_time_series_and_table_data_/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pipedream/nasdaq_data_link_time_series_and_table_data_",
-  "version": "0.0.3",
+  "version": "0.1.0",
   "description": "Pipedream Nasdaq Data Link (Time Series and Table data) Components",
   "main": "nasdaq_data_link_time_series_and_table_data_.app.mjs",
   "keywords": [
@@ -11,5 +11,8 @@
   "author": "Pipedream <support@pipedream.com> (https://pipedream.com/)",
   "publishConfig": {
     "access": "public"
+  },
+  "dependencies": {
+    "@pipedream/platform": "^3.1.0"
   }
 }