diff --git a/components/memento_database/actions/create-entry/create-entry.mjs b/components/memento_database/actions/create-entry/create-entry.mjs new file mode 100644 index 0000000000000..0b4ee9d87fb88 --- /dev/null +++ b/components/memento_database/actions/create-entry/create-entry.mjs @@ -0,0 +1,41 @@ +import mementoDatabase from "../../memento_database.app.mjs"; +import { parseJson } from "../../common/utils.mjs"; + +export default { + key: "memento_database-create-entry", + name: "Create Entry", + description: "Create an entry in a library on Memento Database. [See the documentation](https://mementodatabase.docs.apiary.io/#reference/0/entries/create-a-new-entry)", + version: "0.0.1", + type: "action", + annotations: { + destructiveHint: false, + openWorldHint: true, + readOnlyHint: false, + }, + props: { + mementoDatabase, + libraryId: { + propDefinition: [ + mementoDatabase, + "libraryId", + ], + }, + fields: { + propDefinition: [ + mementoDatabase, + "fields", + ], + }, + }, + async run({ $ }) { + const entry = await this.mementoDatabase.createEntry({ + $, + libraryId: this.libraryId, + data: { + fields: parseJson(this.fields), + }, + }); + $.export("$summary", `Successfully created entry with ID: ${entry.id}`); + return entry; + }, +}; diff --git a/components/memento_database/actions/get-library/get-library.mjs b/components/memento_database/actions/get-library/get-library.mjs new file mode 100644 index 0000000000000..c6122a956f71b --- /dev/null +++ b/components/memento_database/actions/get-library/get-library.mjs @@ -0,0 +1,31 @@ +import mementoDatabase from "../../memento_database.app.mjs"; + +export default { + key: "memento_database-get-library", + name: "Get Library", + description: "Get a library by ID on Memento Database. [See the documentation](https://mementodatabase.docs.apiary.io/#reference/0/library/get-a-library)", + version: "0.0.1", + type: "action", + annotations: { + destructiveHint: false, + openWorldHint: true, + readOnlyHint: true, + }, + props: { + mementoDatabase, + libraryId: { + propDefinition: [ + mementoDatabase, + "libraryId", + ], + }, + }, + async run({ $ }) { + const library = await this.mementoDatabase.getLibrary({ + $, + libraryId: this.libraryId, + }); + $.export("$summary", `Successfully retrieved library with ID: ${library.id}`); + return library; + }, +}; diff --git a/components/memento_database/actions/list-entries/list-entries.mjs b/components/memento_database/actions/list-entries/list-entries.mjs new file mode 100644 index 0000000000000..034096c51ee26 --- /dev/null +++ b/components/memento_database/actions/list-entries/list-entries.mjs @@ -0,0 +1,63 @@ +import mementoDatabase from "../../memento_database.app.mjs"; + +export default { + key: "memento_database-list-entries", + name: "List Entries", + description: "List entries in a library on Memento Database. [See the documentation](https://mementodatabase.docs.apiary.io/#reference/0/entries/list-entries-on-a-library)", + version: "0.0.1", + type: "action", + annotations: { + destructiveHint: false, + openWorldHint: true, + readOnlyHint: true, + }, + props: { + mementoDatabase, + libraryId: { + propDefinition: [ + mementoDatabase, + "libraryId", + ], + }, + fields: { + type: "string", + label: "Fields", + description: "The comma-separated list of fields ids to include in the response. *all - include all fields.", + default: "all", + }, + startRevision: { + type: "integer", + label: "Start Revision", + description: "Only entries updated/created at or after this revision are returned", + optional: true, + }, + pageToken: { + type: "string", + label: "Page Token", + description: "The token for continuing a previous list request on the next page", + optional: true, + }, + pageSize: { + type: "integer", + label: "Page Size", + description: "The maximum number of entries to return", + optional: true, + }, + }, + async run({ $ }) { + const response = await this.mementoDatabase.listEntries({ + $, + libraryId: this.libraryId, + params: { + fields: this.fields, + startRevision: this.startRevision, + pageSize: this.pageSize, + pageToken: this.pageToken, + }, + }); + $.export("$summary", `Successfully listed ${response.entries.length} entr${response.entries.length === 1 + ? "y" + : "ies"}`); + return response; + }, +}; diff --git a/components/memento_database/actions/update-entry/update-entry.mjs b/components/memento_database/actions/update-entry/update-entry.mjs new file mode 100644 index 0000000000000..7b64228b179ab --- /dev/null +++ b/components/memento_database/actions/update-entry/update-entry.mjs @@ -0,0 +1,51 @@ +import mementoDatabase from "../../memento_database.app.mjs"; +import { parseJson } from "../../common/utils.mjs"; + +export default { + key: "memento_database-update-entry", + name: "Update Entry", + description: "Update an entry in a library on Memento Database. [See the documentation](https://mementodatabase.docs.apiary.io/#reference/0/entry/edit-an-entry)", + version: "0.0.1", + type: "action", + annotations: { + destructiveHint: true, + openWorldHint: true, + readOnlyHint: false, + }, + props: { + mementoDatabase, + libraryId: { + propDefinition: [ + mementoDatabase, + "libraryId", + ], + }, + entryId: { + propDefinition: [ + mementoDatabase, + "entryId", + (c) => ({ + libraryId: c.libraryId, + }), + ], + }, + fields: { + propDefinition: [ + mementoDatabase, + "fields", + ], + }, + }, + async run({ $ }) { + const entry = await this.mementoDatabase.updateEntry({ + $, + libraryId: this.libraryId, + entryId: this.entryId, + data: { + fields: parseJson(this.fields), + }, + }); + $.export("$summary", `Successfully updated entry with ID: ${entry.id}`); + return entry; + }, +}; diff --git a/components/memento_database/common/utils.mjs b/components/memento_database/common/utils.mjs new file mode 100644 index 0000000000000..4ce890a47d852 --- /dev/null +++ b/components/memento_database/common/utils.mjs @@ -0,0 +1,40 @@ +export const parseJson = (input, maxDepth = 100) => { + const seen = new WeakSet(); + const parse = (value) => { + if (maxDepth <= 0) { + return value; + } + if (typeof(value) === "string") { + // Only parse if the string looks like a JSON object or array + const trimmed = value.trim(); + if ( + (trimmed.startsWith("{") && trimmed.endsWith("}")) || + (trimmed.startsWith("[") && trimmed.endsWith("]")) + ) { + try { + return parseJson(JSON.parse(value), maxDepth - 1); + } catch (e) { + return value; + } + } + return value; + } else if (typeof(value) === "object" && value !== null && !Array.isArray(value)) { + if (seen.has(value)) { + return value; + } + seen.add(value); + return Object.entries(value) + .reduce((acc, [ + key, + val, + ]) => Object.assign(acc, { + [key]: parse(val), + }), {}); + } else if (Array.isArray(value)) { + return value.map((item) => parse(item)); + } + return value; + }; + + return parse(input); +}; diff --git a/components/memento_database/memento_database.app.mjs b/components/memento_database/memento_database.app.mjs index ca83e3d61f868..c717977418730 100644 --- a/components/memento_database/memento_database.app.mjs +++ b/components/memento_database/memento_database.app.mjs @@ -1,11 +1,111 @@ +import { axios } from "@pipedream/platform"; + export default { type: "app", app: "memento_database", - propDefinitions: {}, + propDefinitions: { + libraryId: { + type: "string", + label: "Library ID", + description: "The ID of a library", + async options() { + const { libraries } = await this.listLibraries(); + return libraries?.map(({ + id: value, name: label, + }) => ({ + value, + label, + })) || []; + }, + }, + entryId: { + type: "string", + label: "Entry ID", + description: "The ID of an entry", + async options({ + libraryId, prevContext, + }) { + const { + entries, nextPageToken, + } = await this.listEntries({ + libraryId, + params: { + pageToken: prevContext?.pageToken, + }, + }); + const options = entries?.map(({ id }) => ({ + value: id, + label: `Entry ${id}`, + })) || []; + return { + options, + context: { + pageToken: nextPageToken, + }, + }; + }, + }, + fields: { + type: "string[]", + label: "Fields", + description: "An array of objects containing the fields of the entry. Note: The field IDs must already exist in the library. Example: `[{ \"id\": 1, \"value\": \"Record 1\" }, { \"id\": 2, \"value\": 1000 }]`", + }, + }, methods: { - // this.$auth contains connected account data - authKeys() { - console.log(Object.keys(this.$auth)); + _baseUrl() { + return "https://api.mementodatabase.com/v1"; + }, + _makeRequest({ + $ = this, path, params, ...opts + }) { + return axios($, { + url: `${this._baseUrl()}${path}`, + params: { + ...params, + token: `${this.$auth.api_token}`, + }, + ...opts, + }); + }, + getLibrary({ + libraryId, ...opts + }) { + return this._makeRequest({ + path: `/libraries/${libraryId}`, + ...opts, + }); + }, + listLibraries(opts = {}) { + return this._makeRequest({ + path: "/libraries", + ...opts, + }); + }, + listEntries({ + libraryId, ...opts + }) { + return this._makeRequest({ + path: `/libraries/${libraryId}/entries`, + ...opts, + }); + }, + createEntry({ + libraryId, ...opts + }) { + return this._makeRequest({ + method: "POST", + path: `/libraries/${libraryId}/entries`, + ...opts, + }); + }, + updateEntry({ + libraryId, entryId, ...opts + }) { + return this._makeRequest({ + method: "PATCH", + path: `/libraries/${libraryId}/entries/${entryId}`, + ...opts, + }); }, }, }; diff --git a/components/memento_database/package.json b/components/memento_database/package.json index c4c82fe77f607..fa26c4fbcfce7 100644 --- a/components/memento_database/package.json +++ b/components/memento_database/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/memento_database", - "version": "0.0.1", + "version": "0.1.0", "description": "Pipedream Memento Database Components", "main": "memento_database.app.mjs", "keywords": [ @@ -11,5 +11,8 @@ "author": "Pipedream (https://pipedream.com/)", "publishConfig": { "access": "public" + }, + "dependencies": { + "@pipedream/platform": "^3.1.1" } -} \ No newline at end of file +} diff --git a/components/memento_database/sources/common/base-polling.mjs b/components/memento_database/sources/common/base-polling.mjs new file mode 100644 index 0000000000000..0aa6b430fc9f9 --- /dev/null +++ b/components/memento_database/sources/common/base-polling.mjs @@ -0,0 +1,118 @@ +import mementoDatabase from "../../memento_database.app.mjs"; +import { + DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, ConfigurationError, +} from "@pipedream/platform"; + +export default { + props: { + mementoDatabase, + db: "$.service.db", + timer: { + type: "$.interface.timer", + default: { + intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, + }, + }, + libraryId: { + propDefinition: [ + mementoDatabase, + "libraryId", + ], + }, + }, + methods: { + _getLastRevision() { + return this.db.get("lastRevision") || 1; + }, + _setLastRevision(lastRevision) { + this.db.set("lastRevision", lastRevision); + }, + _getLastTs() { + return this.db.get("lastTs") || 0; + }, + _setLastTs(lastTs) { + this.db.set("lastTs", lastTs); + }, + getArgs() { + return { + libraryId: this.libraryId, + params: { + fields: "all", + }, + }; + }, + isRelevant() { + return true; + }, + getTsField() { + throw new ConfigurationError("getTsField must be implemented"); + }, + getResourceFn() { + throw new ConfigurationError("getResourceFn must be implemented"); + }, + generateMeta() { + throw new ConfigurationError("generateMeta must be implemented"); + }, + async processEvent(max) { + const lastTs = this._getLastTs(); + let maxTs = lastTs; + + const resourceFn = this.getResourceFn(); + let args = this.getArgs(); + const tsField = this.getTsField(); + + args = { + ...args, + params: { + ...args?.params, + pageSize: 100, + startRevision: this._getLastRevision(), + }, + }; + let hasMore = true; + let nextRevision = args.params.startRevision; + const items = []; + + do { + const { + entries, nextPageToken, revision, + } = await resourceFn(args); + for (const entry of entries) { + const ts = Date.parse(entry[tsField]); + if (ts > lastTs) { + maxTs = Math.max(ts, maxTs); + if (this.isRelevant(entry)) { + items.push(entry); + } + } + } + nextRevision = revision; + if (!entries?.length) { + break; + } + hasMore = entries.length === args.params.pageSize; + args.params.pageToken = nextPageToken; + } while (hasMore); + + this._setLastRevision(nextRevision); + this._setLastTs(maxTs); + + if (max && items.length > max) { + items.length = max; + } + + items.forEach((item) => { + const meta = this.generateMeta(item); + this.$emit(item, meta); + }); + }, + }, + hooks: { + async deploy() { + await this.processEvent(10); + }, + }, + async run() { + await this.processEvent(); + }, +}; diff --git a/components/memento_database/sources/entry-updated/entry-updated.mjs b/components/memento_database/sources/entry-updated/entry-updated.mjs new file mode 100644 index 0000000000000..0b22f7a6554fc --- /dev/null +++ b/components/memento_database/sources/entry-updated/entry-updated.mjs @@ -0,0 +1,30 @@ +import common from "../common/base-polling.mjs"; + +export default { + ...common, + key: "memento_database-entry-updated", + name: "Entry Updated", + description: "Emit new event when an entry is updated in Memento Database. [See the documentation](https://mementodatabase.docs.apiary.io/#reference/0/entries/list-entries-on-a-library)", + version: "0.0.1", + type: "source", + methods: { + ...common.methods, + getTsField() { + return "modifiedTime"; + }, + getResourceFn() { + return this.mementoDatabase.listEntries; + }, + isRelevant(entry) { + return entry.modifiedTime !== entry.createdTime; + }, + generateMeta(entry) { + const ts = Date.parse(entry[this.getTsField()]); + return { + id: `${entry.id}-${ts}`, + summary: `Entry Updated with ID: ${entry.id}`, + ts, + }; + }, + }, +}; diff --git a/components/memento_database/sources/new-entry-created/new-entry-created.mjs b/components/memento_database/sources/new-entry-created/new-entry-created.mjs new file mode 100644 index 0000000000000..36cfc30e8be74 --- /dev/null +++ b/components/memento_database/sources/new-entry-created/new-entry-created.mjs @@ -0,0 +1,26 @@ +import common from "../common/base-polling.mjs"; + +export default { + ...common, + key: "memento_database-new-entry-created", + name: "New Entry Created", + description: "Emit new event when a new entry is created in Memento Database. [See the documentation](https://mementodatabase.docs.apiary.io/#reference/0/entries/list-entries-on-a-library)", + version: "0.0.1", + type: "source", + methods: { + ...common.methods, + getTsField() { + return "createdTime"; + }, + getResourceFn() { + return this.mementoDatabase.listEntries; + }, + generateMeta(entry) { + return { + id: entry.id, + summary: `New Entry with ID: ${entry.id}`, + ts: Date.parse(entry[this.getTsField()]), + }; + }, + }, +}; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 666c3870d1b12..0f7349d6fc321 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2168,8 +2168,7 @@ importers: specifier: ^1.6.8 version: 1.6.8 - components/browser_use: - specifiers: {} + components/browser_use: {} components/browseract: {} @@ -9137,7 +9136,11 @@ importers: components/membervault: {} - components/memento_database: {} + components/memento_database: + dependencies: + '@pipedream/platform': + specifier: ^3.1.1 + version: 3.1.1 components/memix: dependencies: @@ -31830,17 +31833,17 @@ packages: superagent@3.8.1: resolution: {integrity: sha512-VMBFLYgFuRdfeNQSMLbxGSLfmXL/xc+OO+BZp41Za/NRDBet/BNbkRJrYzCUu0u4GU0i/ml2dtT8b9qgkw9z6Q==} engines: {node: '>= 4.0'} - deprecated: Please upgrade to v9.0.0+ as we have fixed a public vulnerability with formidable dependency. Note that v9.0.0+ requires Node.js v14.18.0+. See https://github.com/ladjs/superagent/pull/1800 for insight. This project is supported and maintained by the team at Forward Email @ https://forwardemail.net + deprecated: Please upgrade to superagent v10.2.2+, see release notes at https://github.com/forwardemail/superagent/releases/tag/v10.2.2 - maintenance is supported by Forward Email @ https://forwardemail.net superagent@4.1.0: resolution: {integrity: sha512-FT3QLMasz0YyCd4uIi5HNe+3t/onxMyEho7C3PSqmti3Twgy2rXT4fmkTz6wRL6bTF4uzPcfkUCa8u4JWHw8Ag==} engines: {node: '>= 6.0'} - deprecated: Please upgrade to v9.0.0+ as we have fixed a public vulnerability with formidable dependency. Note that v9.0.0+ requires Node.js v14.18.0+. See https://github.com/ladjs/superagent/pull/1800 for insight. This project is supported and maintained by the team at Forward Email @ https://forwardemail.net + deprecated: Please upgrade to superagent v10.2.2+, see release notes at https://github.com/forwardemail/superagent/releases/tag/v10.2.2 - maintenance is supported by Forward Email @ https://forwardemail.net superagent@5.3.1: resolution: {integrity: sha512-wjJ/MoTid2/RuGCOFtlacyGNxN9QLMgcpYLDQlWFIhhdJ93kNscFonGvrpAHSCVjRVj++DGCglocF7Aej1KHvQ==} engines: {node: '>= 7.0.0'} - deprecated: Please upgrade to v9.0.0+ as we have fixed a public vulnerability with formidable dependency. Note that v9.0.0+ requires Node.js v14.18.0+. See https://github.com/ladjs/superagent/pull/1800 for insight. This project is supported and maintained by the team at Forward Email @ https://forwardemail.net + deprecated: Please upgrade to superagent v10.2.2+, see release notes at https://github.com/forwardemail/superagent/releases/tag/v10.2.2 - maintenance is supported by Forward Email @ https://forwardemail.net supports-color@10.2.2: resolution: {integrity: sha512-SS+jx45GF1QjgEXQx4NJZV9ImqmO2NPz5FNsIHrsDjh2YsHnawpan7SNQ1o8NuhrbHZy9AZhIoCUiCeaW/C80g==} @@ -39531,7 +39534,6 @@ snapshots: transitivePeerDependencies: - rolldown - rollup - - supports-color '@putout/operator-parens@2.0.0(rolldown@1.0.0-beta.9)(rollup@4.53.2)': dependencies: