diff --git a/components/capturekit/actions/capture-screenshot/capture-screenshot.mjs b/components/capturekit/actions/capture-screenshot/capture-screenshot.mjs new file mode 100644 index 0000000000000..2fc3cd2588c15 --- /dev/null +++ b/components/capturekit/actions/capture-screenshot/capture-screenshot.mjs @@ -0,0 +1,78 @@ +import app from "../../capturekit.app.mjs"; +import fs from "fs"; +import path from "path"; + +export default { + key: "capturekit-capture-screenshot", + name: "Capture Screenshot", + description: "Capture a high-quality image of any webpage. [See the documentation](https://docs.capturekit.dev/api-reference/screenshot-api)", + version: "0.0.1", + type: "action", + props: { + app, + url: { + propDefinition: [ + app, + "url", + ], + }, + format: { + propDefinition: [ + app, + "format", + ], + }, + device: { + propDefinition: [ + app, + "device", + ], + }, + cache: { + propDefinition: [ + app, + "cache", + ], + }, + fullPageScroll: { + propDefinition: [ + app, + "fullPageScroll", + ], + }, + fileName: { + propDefinition: [ + app, + "fileName", + ], + }, + syncDir: { + type: "dir", + accessMode: "write", + sync: true, + }, + }, + async run({ $ }) { + const response = await this.app.captureScreenshot({ + $, + params: { + url: this.url, + format: this.format, + device: this.device, + cache: this.cache, + full_page_scroll: this.fullPageScroll, + }, + responseType: "arraybuffer", + }); + + const fileExtension = this.format || "png"; + const fileName = `${this.fileName}.${fileExtension}`; + const filePath = path.join("/tmp", fileName); + + await fs.promises.writeFile(filePath, response); + + $.export("$summary", `Screenshot successfully saved to: ${filePath}`); + + return filePath; + }, +}; diff --git a/components/capturekit/actions/scrape-content/scrape-content.mjs b/components/capturekit/actions/scrape-content/scrape-content.mjs new file mode 100644 index 0000000000000..421fed01b6e03 --- /dev/null +++ b/components/capturekit/actions/scrape-content/scrape-content.mjs @@ -0,0 +1,63 @@ +import app from "../../capturekit.app.mjs"; + +export default { + key: "capturekit-scrape-content", + name: "Scrape Content", + description: "Extract structured data from any webpage, including metadata, links, and raw HTML. [See the documentation](https://docs.capturekit.dev/api-reference/content-api)", + version: "0.0.1", + type: "action", + props: { + app, + url: { + propDefinition: [ + app, + "url", + ], + }, + includeHtml: { + propDefinition: [ + app, + "includeHtml", + ], + }, + useDefuddle: { + propDefinition: [ + app, + "useDefuddle", + ], + }, + selector: { + propDefinition: [ + app, + "selector", + ], + }, + removeSelectors: { + propDefinition: [ + app, + "removeSelectors", + ], + }, + blockUrls: { + propDefinition: [ + app, + "blockUrls", + ], + }, + }, + async run({ $ }) { + const response = await this.app.scrapeContent({ + $, + params: { + url: this.url, + include_html: this.includeHtml, + use_defuddle: this.useDefuddle, + selector: this.selector, + remove_selectors: (this.removeSelectors || []).join(","), + block_urls: (this.blockUrls || []).join(","), + }, + }); + $.export("$summary", "Successfully sent the Scrape Content request"); + return response; + }, +}; diff --git a/components/capturekit/capturekit.app.mjs b/components/capturekit/capturekit.app.mjs index 7eb6b968ef23d..063d1832b2213 100644 --- a/components/capturekit/capturekit.app.mjs +++ b/components/capturekit/capturekit.app.mjs @@ -1,11 +1,134 @@ +import { axios } from "@pipedream/platform"; + export default { type: "app", app: "capturekit", - propDefinitions: {}, + propDefinitions: { + url: { + type: "string", + label: "URL", + description: "Target webpage to capture", + }, + format: { + type: "string", + label: "Format", + description: "The output format of the screenshot", + optional: true, + options: [ + "webp", + "jpeg", + "jpg", + "png", + "pdf", + ], + }, + device: { + type: "string", + label: "Device", + description: "Device type used for emulation (desktop or mobile)", + optional: true, + options: [ + "iphone_14_pro_max", + "iphone_14_pro", + "iphone_13_pro_max", + "iphone_13_mini", + "galaxy_s23_ultra", + "galaxy_s23", + "galaxy_fold4", + "pixel_7_pro", + "pixel_6a", + "redmi_note_12_pro", + "redmi_note_11", + "huawei_p60_pro", + "huawei_mate_50_pro", + "iphone_x", + "iphone_12", + "pixel_5", + "galaxy_s8", + "ipad", + ], + }, + cache: { + type: "boolean", + label: "Cache", + description: "Enable or disable response caching", + optional: true, + }, + fullPageScroll: { + type: "boolean", + label: "Full Page Scroll", + description: "Scroll through the entire page before capturing", + optional: true, + }, + includeHtml: { + type: "boolean", + label: "Include HTML", + description: "Return the raw HTML content in the response", + optional: true, + }, + useDefuddle: { + type: "boolean", + label: "Use Defuddle", + description: "Use Defuddle to clean and extract the main content from web pages", + optional: true, + }, + selector: { + type: "string", + label: "Selector", + description: "Capture a specific element on the page instead of the full viewport", + optional: true, + }, + removeSelectors: { + type: "string[]", + label: "Remove Selectors", + description: "A list of elements to hide before capturing", + optional: true, + }, + blockUrls: { + type: "string[]", + label: "Block URLs", + description: "A ist of URL patterns to block. You can specify URLs, domains, or simple patterns, e.g.: `.example.com/`", + optional: true, + }, + fileName: { + type: "string", + label: "File Name", + description: "Name of the screenshot file that will be saved on /tmp", + }, + }, methods: { - // this.$auth contains connected account data - authKeys() { - console.log(Object.keys(this.$auth)); + _baseUrl() { + return "https://api.capturekit.dev"; + }, + async _makeRequest(opts = {}) { + const { + $ = this, + path, + headers, + ...otherOpts + } = opts; + return axios($, { + ...otherOpts, + url: this._baseUrl() + path, + headers: { + "x-access-key": `${this.$auth.access_key}`, + ...headers, + }, + }); + }, + + async captureScreenshot(args = {}) { + return this._makeRequest({ + path: "/capture", + ...args, + }); + }, + + async scrapeContent(args = {}) { + return this._makeRequest({ + path: "/content", + ...args, + }); }, }, }; diff --git a/components/capturekit/package.json b/components/capturekit/package.json index e1de69709c8a1..aac0241fe2b47 100644 --- a/components/capturekit/package.json +++ b/components/capturekit/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/capturekit", - "version": "0.0.1", + "version": "0.1.0", "description": "Pipedream CaptureKit Components", "main": "capturekit.app.mjs", "keywords": [ @@ -11,5 +11,8 @@ "author": "Pipedream (https://pipedream.com/)", "publishConfig": { "access": "public" + }, + "dependencies": { + "@pipedream/platform": "^3.1.0" } -} \ No newline at end of file +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 620f059274e72..9089326ba085c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2193,7 +2193,11 @@ importers: components/captaindata: {} - components/capturekit: {} + components/capturekit: + dependencies: + '@pipedream/platform': + specifier: ^3.1.0 + version: 3.1.0 components/carbone: {}