Merge pull request #1631 from fletch-ai/feature/airtable-include-fields

Feature/airtable include fields

packages/components/nodes/documentloaders/Airtable/Airtable.ts

@@ -20,7 +20,7 @@ class Airtable_DocumentLoaders implements INode {
     constructor() {
         this.label = 'Airtable'
         this.name = 'airtable'
-        this.version = 2.0
+        this.version = 3.0
         this.type = 'Document'
         this.icon = 'airtable.svg'
         this.category = 'Document Loaders'
@@ -64,10 +64,21 @@ class Airtable_DocumentLoaders implements INode {
                     'If your view URL looks like: https://airtable.com/app11RobdGoX0YNsC/tblJdmvbrgizbYICO/viw9UrP77Id0CE4ee, viw9UrP77Id0CE4ee is the view id',
                 optional: true
             },
+            {
+                label: 'Include Only Fields',
+                name: 'fields',
+                type: 'string',
+                placeholder: 'Name, Assignee, fld1u0qUz0SoOQ9Gg, fldew39v6LBN5CjUl',
+                optional: true,
+                additionalParams: true,
+                description:
+                    'Comma-separated list of field names or IDs to include. If empty, then ALL fields are used. Use field IDs if field names contain commas.'
+            },
             {
                 label: 'Return All',
                 name: 'returnAll',
                 type: 'boolean',
+                optional: true,
                 default: true,
                 additionalParams: true,
                 description: 'If all results should be returned or only up to a given limit'
@@ -76,9 +87,10 @@ class Airtable_DocumentLoaders implements INode {
                 label: 'Limit',
                 name: 'limit',
                 type: 'number',
+                optional: true,
                 default: 100,
                 additionalParams: true,
-                description: 'Number of results to return'
+                description: 'Number of results to return. Ignored when Return All is enabled.'
             },
             {
                 label: 'Metadata',
@@ -93,6 +105,8 @@ class Airtable_DocumentLoaders implements INode {
         const baseId = nodeData.inputs?.baseId as string
         const tableId = nodeData.inputs?.tableId as string
         const viewId = nodeData.inputs?.viewId as string
+        const fieldsInput = nodeData.inputs?.fields as string
+        const fields = fieldsInput ? fieldsInput.split(',').map((field) => field.trim()) : []
         const returnAll = nodeData.inputs?.returnAll as boolean
         const limit = nodeData.inputs?.limit as string
         const textSplitter = nodeData.inputs?.textSplitter as TextSplitter
@@ -105,6 +119,7 @@ class Airtable_DocumentLoaders implements INode {
             baseId,
             tableId,
             viewId,
+            fields,
             returnAll,
             accessToken,
             limit: limit ? parseInt(limit, 10) : 100
@@ -112,6 +127,10 @@ class Airtable_DocumentLoaders implements INode {
 
         const loader = new AirtableLoader(airtableOptions)
 
+        if (!baseId || !tableId) {
+            throw new Error('Base ID and Table ID must be provided.')
+        }
+
         let docs = []
 
         if (textSplitter) {
@@ -145,10 +164,18 @@ interface AirtableLoaderParams {
     tableId: string
     accessToken: string
     viewId?: string
+    fields?: string[]
     limit?: number
     returnAll?: boolean
 }
 
+interface AirtableLoaderRequest {
+    maxRecords?: number
+    view: string | undefined
+    fields?: string[]
+    offset?: string
+}
+
 interface AirtableLoaderResponse {
     records: AirtableLoaderPage[]
     offset?: string
@@ -167,17 +194,20 @@ class AirtableLoader extends BaseDocumentLoader {
 
     public readonly viewId?: string
 
+    public readonly fields: string[]
+
     public readonly accessToken: string
 
     public readonly limit: number
 
     public readonly returnAll: boolean
 
-    constructor({ baseId, tableId, viewId, accessToken, limit = 100, returnAll = false }: AirtableLoaderParams) {
+    constructor({ baseId, tableId, viewId, fields = [], accessToken, limit = 100, returnAll = false }: AirtableLoaderParams) {
         super()
         this.baseId = baseId
         this.tableId = tableId
         this.viewId = viewId
+        this.fields = fields
         this.accessToken = accessToken
         this.limit = limit
         this.returnAll = returnAll
@@ -190,17 +220,21 @@ class AirtableLoader extends BaseDocumentLoader {
         return this.loadLimit()
     }
 
-    protected async fetchAirtableData(url: string, params: ICommonObject): Promise<AirtableLoaderResponse> {
+    protected async fetchAirtableData(url: string, data: AirtableLoaderRequest): Promise<AirtableLoaderResponse> {
         try {
             const headers = {
                 Authorization: `Bearer ${this.accessToken}`,
                 'Content-Type': 'application/json',
                 Accept: 'application/json'
             }
-            const response = await axios.get(url, { params, headers })
+            const response = await axios.post(url, data, { headers })
             return response.data
         } catch (error) {
-            throw new Error(`Failed to fetch ${url} from Airtable: ${error}`)
+            if (axios.isAxiosError(error)) {
+                throw new Error(`Failed to fetch ${url} from Airtable: ${error.message}, status: ${error.response?.status}`)
+            } else {
+                throw new Error(`Failed to fetch ${url} from Airtable: ${error}`)
+            }
         }
     }
 
@@ -218,24 +252,53 @@ class AirtableLoader extends BaseDocumentLoader {
     }
 
     private async loadLimit(): Promise<Document[]> {
-        const params = { maxRecords: this.limit, view: this.viewId }
+        let data: AirtableLoaderRequest = {
-        const data = await this.fetchAirtableData(`https://api.airtable.com/v0/${this.baseId}/${this.tableId}`, params)
+            maxRecords: this.limit,
-        if (data.records.length === 0) {
+            view: this.viewId
-            return []
         }
-        return data.records.map((page) => this.createDocumentFromPage(page))
+
+        if (this.fields.length > 0) {
+            data.fields = this.fields
+        }
+
+        let response: AirtableLoaderResponse
+        let returnPages: AirtableLoaderPage[] = []
+
+        // Paginate if the user specifies a limit > 100 (like 200) but not return all.
+        do {
+            response = await this.fetchAirtableData(`https://api.airtable.com/v0/${this.baseId}/${this.tableId}/listRecords`, data)
+            returnPages.push(...response.records)
+            data.offset = response.offset
+
+            // Stop if we have fetched enough records
+            if (returnPages.length >= this.limit) break
+        } while (response.offset !== undefined)
+
+        // Truncate array to the limit if necessary
+        if (returnPages.length > this.limit) {
+            returnPages.length = this.limit
+        }
+
+        return returnPages.map((page) => this.createDocumentFromPage(page))
     }
 
     private async loadAll(): Promise<Document[]> {
-        const params: ICommonObject = { pageSize: 100, view: this.viewId }
+        let data: AirtableLoaderRequest = {
-        let data: AirtableLoaderResponse
+            view: this.viewId
+        }
+
+        if (this.fields.length > 0) {
+            data.fields = this.fields
+        }
+
+        let response: AirtableLoaderResponse
         let returnPages: AirtableLoaderPage[] = []
 
         do {
-            data = await this.fetchAirtableData(`https://api.airtable.com/v0/${this.baseId}/${this.tableId}`, params)
+            response = await this.fetchAirtableData(`https://api.airtable.com/v0/${this.baseId}/${this.tableId}/listRecords`, data)
-            returnPages.push.apply(returnPages, data.records)
+            returnPages.push(...response.records)
-            params.offset = data.offset
+            data.offset = response.offset
-        } while (data.offset !== undefined)
+        } while (response.offset !== undefined)
         return returnPages.map((page) => this.createDocumentFromPage(page))
     }
 }