From c0fc41a086af50c1782701656c7af51063c640fc Mon Sep 17 00:00:00 2001 From: Grant Linville Date: Mon, 4 Nov 2024 21:46:04 -0500 Subject: [PATCH 1/5] chore: update for dataset rewrite Signed-off-by: Grant Linville --- src/gptscript.ts | 113 ++++++++++------------------------------ tests/gptscript.test.ts | 98 +++++++++++----------------------- 2 files changed, 58 insertions(+), 153 deletions(-) diff --git a/src/gptscript.ts b/src/gptscript.ts index 406f702..cc46ddf 100644 --- a/src/gptscript.ts +++ b/src/gptscript.ts @@ -12,7 +12,7 @@ export interface GlobalOpts { BaseURL?: string DefaultModel?: string DefaultModelProvider?: string - DatasetToolRepo?: string + DatasetTool?: string WorkspaceTool?: string Env?: string[] } @@ -386,106 +386,61 @@ export class GPTScript { }) } - // Dataset methods - - async listDatasets(workspaceID: string): Promise> { - if (workspaceID == "") { - workspaceID = process.env.GPTSCRIPT_WORKSPACE_ID ?? "" - } - + // returns an array of dataset IDs + async listDatasets(): Promise> { const result = await this.runBasicCommand("datasets", { - workspaceID: workspaceID, - datasetToolRepo: this.opts.DatasetToolRepo ?? "", - env: this.opts.Env - }) - return JSON.parse(result) as Array - } - - async createDataset(workspaceID: string, name: string, description: string): Promise { - if (workspaceID == "") { - workspaceID = process.env.GPTSCRIPT_WORKSPACE_ID ?? "" - } - - const result = await this.runBasicCommand("datasets/create", { - input: JSON.stringify({datasetName: name, datasetDescription: description}), - workspaceID: workspaceID, - datasetToolRepo: this.opts.DatasetToolRepo ?? "", + input: JSON.stringify({"workspaceID": process.env.GPTSCRIPT_WORKSPACE_ID}), + datasetTool: this.opts.DatasetTool ?? "", env: this.opts.Env }) - return JSON.parse(result) as Dataset + return JSON.parse(result) as Array } - async addDatasetElement(workspaceID: string, datasetID: string, elementName: string, elementDescription: string, elementContent: ArrayBuffer): Promise { - if (workspaceID == "") { - workspaceID = process.env.GPTSCRIPT_WORKSPACE_ID ?? "" - } - - const result = await this.runBasicCommand("datasets/add-element", { - input: JSON.stringify({ - datasetID, - elementName: elementName, - elementDescription: elementDescription, - elementContent: Buffer.from(elementContent).toString("base64") - }), - workspaceID: workspaceID, - datasetToolRepo: this.opts.DatasetToolRepo ?? "", - env: this.opts.Env - }) - return JSON.parse(result) as DatasetElementMeta - } - - async addDatasetElements(workspaceID: string, datasetID: string, elements: Array) { - if (workspaceID === "") { - workspaceID = process.env.GPTSCRIPT_WORKSPACE_ID ?? "" - } - + async addDatasetElements(elements: Array, datasetID?: string) { const serializableElements = elements.map(e => { return { name: e.name, description: e.description, - contents: Buffer.from(e.contents).toString("base64") + contents: e.contents, + binaryContents: Buffer.from(e.binaryContents ?? Buffer.from("")).toString("base64") } }) return await this.runBasicCommand("datasets/add-elements", { - input: JSON.stringify({datasetID, elements: serializableElements}), - workspaceID: workspaceID, - datasetToolRepo: this.opts.DatasetToolRepo ?? "", + input: JSON.stringify({ + workspaceID: process.env.GPTSCRIPT_WORKSPACE_ID, + datasetID: datasetID ?? "", + elements: serializableElements + }), + datasetTool: this.opts.DatasetTool ?? "", env: this.opts.Env, }) } - async listDatasetElements(workspaceID: string, datasetID: string): Promise> { - if (workspaceID == "") { - workspaceID = process.env.GPTSCRIPT_WORKSPACE_ID ?? "" - } - + async listDatasetElements(datasetID: string): Promise> { const result = await this.runBasicCommand("datasets/list-elements", { - input: JSON.stringify({datasetID}), - workspaceID: workspaceID, - datasetToolRepo: this.opts.DatasetToolRepo ?? "", + input: JSON.stringify({workspaceID: process.env.GPTSCRIPT_WORKSPACE_ID, datasetID}), + datasetTool: this.opts.DatasetTool ?? "", env: this.opts.Env }) return JSON.parse(result) as Array } - async getDatasetElement(workspaceID: string, datasetID: string, elementName: string): Promise { - if (workspaceID == "") { - workspaceID = process.env.GPTSCRIPT_WORKSPACE_ID ?? "" - } - + async getDatasetElement(datasetID: string, elementName: string): Promise { const result = await this.runBasicCommand("datasets/get-element", { - input: JSON.stringify({datasetID, element: elementName}), - workspaceID: workspaceID, - datasetToolRepo: this.opts.DatasetToolRepo ?? "", + input: JSON.stringify({workspaceID: process.env.GPTSCRIPT_WORKSPACE_ID, datasetID, name: elementName}), + datasetTool: this.opts.DatasetTool ?? "", env: this.opts.Env }) + console.log(result) + const element = JSON.parse(result) return { name: element.name, description: element.description, - contents: Buffer.from(element.contents, "base64") + contents: element.contents, + binaryContents: Buffer.from(element.binaryContents, "base64") } } @@ -1312,8 +1267,6 @@ function jsonToCredential(cred: string): Credential { } } -// Dataset types - export interface DatasetElementMeta { name: string description: string @@ -1322,18 +1275,6 @@ export interface DatasetElementMeta { export interface DatasetElement { name: string description: string - contents: ArrayBuffer -} - -export interface DatasetMeta { - id: string - name: string - description: string -} - -export interface Dataset { - id: string - name: string - description: string - elements: Record + contents?: string + binaryContents?: ArrayBuffer } diff --git a/tests/gptscript.test.ts b/tests/gptscript.test.ts index fa3dec9..b406dda 100644 --- a/tests/gptscript.test.ts +++ b/tests/gptscript.test.ts @@ -887,112 +887,76 @@ describe("gptscript module", () => { }, 20000) test("dataset operations", async () => { - const datasetName = "test-" + randomBytes(10).toString("hex") - const workspaceID = await g.createWorkspace("directory") + process.env.GPTSCRIPT_WORKSPACE_ID = await g.createWorkspace("directory") let datasetID: string - // Create + // Create and add two elements try { - const dataset = await g.createDataset(workspaceID, datasetName, "a test dataset") - expect(dataset).toBeDefined() - expect(dataset.name).toEqual(datasetName) - expect(dataset.description).toEqual("a test dataset") - expect(dataset.id.length).toBeGreaterThan(0) - expect(dataset.elements).toEqual({}) - datasetID = dataset.id + datasetID = await g.addDatasetElements([ + { + name: "element1", + description: "", + contents: "this is element 1 contents" + }, + { + name: "element2", + description: "a description", + binaryContents: Buffer.from("this is element 2 contents") + } + ]) } catch (e) { throw new Error("failed to create dataset: " + e) } - // Add elements - try { - const e1 = await g.addDatasetElement( - workspaceID, - datasetID, - "element1", - "", - Buffer.from("this is element 1 contents") - ) - expect(e1.name).toEqual("element1") - expect(e1.description).toEqual("") - - const e2 = await g.addDatasetElement( - workspaceID, - datasetID, - "element2", - "a description", - Buffer.from("this is element 2 contents") - ) - expect(e2.name).toEqual("element2") - expect(e2.description).toEqual("a description") - } catch (e) { - throw new Error("failed to add elements: " + e) - } - - // Add two elements at once. + // Add another element try { - await g.addDatasetElements( - workspaceID, - datasetID, - [ - { - name: "element3", - description: "a description", - contents: Buffer.from("this is element 3 contents") - }, + await g.addDatasetElements([ { - name: "element4", - description: "a description", - contents: Buffer.from("this is element 4 contents") + name: "element3", + description: "a description", + contents: "this is element 3 contents" } - ] - ) + ], datasetID) } catch (e) { throw new Error("failed to add elements: " + e) } // Get elements try { - const e1 = await g.getDatasetElement(workspaceID, datasetID, "element1") + const e1 = await g.getDatasetElement(datasetID, "element1") expect(e1.name).toEqual("element1") - expect(e1.description).toBeUndefined() - expect(e1.contents).toEqual(Buffer.from("this is element 1 contents")) + expect(e1.description).toEqual("") + expect(e1.contents).toEqual("this is element 1 contents") - const e2 = await g.getDatasetElement(workspaceID, datasetID, "element2") + const e2 = await g.getDatasetElement(datasetID, "element2") expect(e2.name).toEqual("element2") expect(e2.description).toEqual("a description") - expect(e2.contents).toEqual(Buffer.from("this is element 2 contents")) + expect(e2.binaryContents).toEqual(Buffer.from("this is element 2 contents")) - const e3 = await g.getDatasetElement(workspaceID, datasetID, "element3") + const e3 = await g.getDatasetElement(datasetID, "element3") expect(e3.name).toEqual("element3") expect(e3.description).toEqual("a description") - expect(e3.contents).toEqual(Buffer.from("this is element 3 contents")) - - const e4 = await g.getDatasetElement(workspaceID, datasetID, "element4") - expect(e4.name).toEqual("element4") - expect(e4.description).toEqual("a description") - expect(e4.contents).toEqual(Buffer.from("this is element 4 contents")) + expect(e3.contents).toEqual("this is element 3 contents") } catch (e) { throw new Error("failed to get elements: " + e) } // List the elements in the dataset try { - const elements = await g.listDatasetElements(workspaceID, datasetID) - expect(elements.length).toEqual(4) + const elements = await g.listDatasetElements(datasetID) + expect(elements.length).toEqual(3) expect(elements.map(e => e.name)).toContain("element1") expect(elements.map(e => e.name)).toContain("element2") expect(elements.map(e => e.name)).toContain("element3") - expect(elements.map(e => e.name)).toContain("element4") } catch (e) { throw new Error("failed to list elements: " + e) } // List datasets try { - const datasets = await g.listDatasets(workspaceID) + const datasets = await g.listDatasets() expect(datasets.length).toBeGreaterThan(0) - expect(datasets.map(d => d.name)).toContain(datasetName) + expect(datasets).toContain(datasetID) } catch (e) { throw new Error("failed to list datasets: " + e) } From de3018a11341d32deb0624efc4aa3fae16daf5fa Mon Sep 17 00:00:00 2001 From: Grant Linville Date: Mon, 4 Nov 2024 21:47:56 -0500 Subject: [PATCH 2/5] remove debug log Signed-off-by: Grant Linville --- src/gptscript.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/gptscript.ts b/src/gptscript.ts index cc46ddf..72442bf 100644 --- a/src/gptscript.ts +++ b/src/gptscript.ts @@ -433,8 +433,6 @@ export class GPTScript { env: this.opts.Env }) - console.log(result) - const element = JSON.parse(result) return { name: element.name, From 49125f86e4a0c2af02b85373fdf647699dfcd58b Mon Sep 17 00:00:00 2001 From: Grant Linville Date: Tue, 5 Nov 2024 16:19:39 -0500 Subject: [PATCH 3/5] workspaceID updates Signed-off-by: Grant Linville --- src/gptscript.ts | 25 ++++++++++++++++--------- tests/gptscript.test.ts | 29 ++++++++++++++++++----------- 2 files changed, 34 insertions(+), 20 deletions(-) diff --git a/src/gptscript.ts b/src/gptscript.ts index 72442bf..98a20fb 100644 --- a/src/gptscript.ts +++ b/src/gptscript.ts @@ -387,16 +387,16 @@ export class GPTScript { } // returns an array of dataset IDs - async listDatasets(): Promise> { + async listDatasets(): Promise> { const result = await this.runBasicCommand("datasets", { - input: JSON.stringify({"workspaceID": process.env.GPTSCRIPT_WORKSPACE_ID}), + input: "{}", datasetTool: this.opts.DatasetTool ?? "", env: this.opts.Env }) - return JSON.parse(result) as Array + return JSON.parse(result) as Array } - async addDatasetElements(elements: Array, datasetID?: string) { + async addDatasetElements(elements: Array, opts: {name?: string, description?: string, datasetID?: string}): Promise { const serializableElements = elements.map(e => { return { name: e.name, @@ -408,18 +408,19 @@ export class GPTScript { return await this.runBasicCommand("datasets/add-elements", { input: JSON.stringify({ - workspaceID: process.env.GPTSCRIPT_WORKSPACE_ID, - datasetID: datasetID ?? "", + name: opts.name ?? "", + description: opts.description ?? "", + datasetID: opts.datasetID ?? "", elements: serializableElements }), datasetTool: this.opts.DatasetTool ?? "", - env: this.opts.Env, + env: this.opts.Env }) } async listDatasetElements(datasetID: string): Promise> { const result = await this.runBasicCommand("datasets/list-elements", { - input: JSON.stringify({workspaceID: process.env.GPTSCRIPT_WORKSPACE_ID, datasetID}), + input: JSON.stringify({datasetID}), datasetTool: this.opts.DatasetTool ?? "", env: this.opts.Env }) @@ -428,7 +429,7 @@ export class GPTScript { async getDatasetElement(datasetID: string, elementName: string): Promise { const result = await this.runBasicCommand("datasets/get-element", { - input: JSON.stringify({workspaceID: process.env.GPTSCRIPT_WORKSPACE_ID, datasetID, name: elementName}), + input: JSON.stringify({datasetID, name: elementName}), datasetTool: this.opts.DatasetTool ?? "", env: this.opts.Env }) @@ -1265,6 +1266,12 @@ function jsonToCredential(cred: string): Credential { } } +export interface DatasetMeta { + id: string + name: string + description: string +} + export interface DatasetElementMeta { name: string description: string diff --git a/tests/gptscript.test.ts b/tests/gptscript.test.ts index b406dda..8206bdc 100644 --- a/tests/gptscript.test.ts +++ b/tests/gptscript.test.ts @@ -887,12 +887,17 @@ describe("gptscript module", () => { }, 20000) test("dataset operations", async () => { - process.env.GPTSCRIPT_WORKSPACE_ID = await g.createWorkspace("directory") + const workspaceID = await g.createWorkspace("directory") + const client = new gptscript.GPTScript({ + APIKey: process.env.OPENAI_API_KEY, + Env: ["GPTSCRIPT_WORKSPACE_ID=" + workspaceID] + }) + let datasetID: string // Create and add two elements try { - datasetID = await g.addDatasetElements([ + datasetID = await client.addDatasetElements([ { name: "element1", description: "", @@ -903,37 +908,37 @@ describe("gptscript module", () => { description: "a description", binaryContents: Buffer.from("this is element 2 contents") } - ]) + ], {name: "test-dataset", description: "a test dataset"}) } catch (e) { throw new Error("failed to create dataset: " + e) } // Add another element try { - await g.addDatasetElements([ + await client.addDatasetElements([ { name: "element3", description: "a description", contents: "this is element 3 contents" } - ], datasetID) + ], {datasetID: datasetID}) } catch (e) { throw new Error("failed to add elements: " + e) } // Get elements try { - const e1 = await g.getDatasetElement(datasetID, "element1") + const e1 = await client.getDatasetElement(datasetID, "element1") expect(e1.name).toEqual("element1") expect(e1.description).toEqual("") expect(e1.contents).toEqual("this is element 1 contents") - const e2 = await g.getDatasetElement(datasetID, "element2") + const e2 = await client.getDatasetElement(datasetID, "element2") expect(e2.name).toEqual("element2") expect(e2.description).toEqual("a description") expect(e2.binaryContents).toEqual(Buffer.from("this is element 2 contents")) - const e3 = await g.getDatasetElement(datasetID, "element3") + const e3 = await client.getDatasetElement(datasetID, "element3") expect(e3.name).toEqual("element3") expect(e3.description).toEqual("a description") expect(e3.contents).toEqual("this is element 3 contents") @@ -943,7 +948,7 @@ describe("gptscript module", () => { // List the elements in the dataset try { - const elements = await g.listDatasetElements(datasetID) + const elements = await client.listDatasetElements(datasetID) expect(elements.length).toEqual(3) expect(elements.map(e => e.name)).toContain("element1") expect(elements.map(e => e.name)).toContain("element2") @@ -954,9 +959,11 @@ describe("gptscript module", () => { // List datasets try { - const datasets = await g.listDatasets() + const datasets = await client.listDatasets() expect(datasets.length).toBeGreaterThan(0) - expect(datasets).toContain(datasetID) + expect(datasets[0].id).toEqual(datasetID) + expect(datasets[0].name).toEqual("test-dataset") + expect(datasets[0].description).toEqual("a test dataset") } catch (e) { throw new Error("failed to list datasets: " + e) } From 8b7cf338c4406446ffd4b47dd7f48cff8b758abc Mon Sep 17 00:00:00 2001 From: Grant Linville Date: Wed, 6 Nov 2024 13:51:26 -0500 Subject: [PATCH 4/5] fix environment Signed-off-by: Grant Linville --- tests/gptscript.test.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/gptscript.test.ts b/tests/gptscript.test.ts index 8206bdc..3c9bf87 100644 --- a/tests/gptscript.test.ts +++ b/tests/gptscript.test.ts @@ -887,10 +887,11 @@ describe("gptscript module", () => { }, 20000) test("dataset operations", async () => { - const workspaceID = await g.createWorkspace("directory") + process.env.GPTSCRIPT_WORKSPACE_ID = await g.createWorkspace("directory") + const client = new gptscript.GPTScript({ APIKey: process.env.OPENAI_API_KEY, - Env: ["GPTSCRIPT_WORKSPACE_ID=" + workspaceID] + Env: Object.entries(process.env).map(([k, v]) => `${k}=${v}`) }) let datasetID: string From 8898263d86c46e9d82696ed468072260affe645e Mon Sep 17 00:00:00 2001 From: Grant Linville Date: Wed, 6 Nov 2024 14:53:14 -0500 Subject: [PATCH 5/5] fix dataset test Signed-off-by: Grant Linville --- src/gptscript.ts | 2 +- tests/gptscript.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/gptscript.ts b/src/gptscript.ts index 98a20fb..d241cb0 100644 --- a/src/gptscript.ts +++ b/src/gptscript.ts @@ -439,7 +439,7 @@ export class GPTScript { name: element.name, description: element.description, contents: element.contents, - binaryContents: Buffer.from(element.binaryContents, "base64") + binaryContents: Buffer.from(element.binaryContents ?? "", "base64") } } diff --git a/tests/gptscript.test.ts b/tests/gptscript.test.ts index 3c9bf87..077107c 100644 --- a/tests/gptscript.test.ts +++ b/tests/gptscript.test.ts @@ -931,7 +931,7 @@ describe("gptscript module", () => { try { const e1 = await client.getDatasetElement(datasetID, "element1") expect(e1.name).toEqual("element1") - expect(e1.description).toEqual("") + expect(e1.description).toBeUndefined() expect(e1.contents).toEqual("this is element 1 contents") const e2 = await client.getDatasetElement(datasetID, "element2")