From 9b71f683fffe916baced5693cbe0f070dd043d12 Mon Sep 17 00:00:00 2001
From: Darien Kindlund <darien@kindlund.com>
Date: Sat, 27 Jan 2024 13:58:20 -0500
Subject: [PATCH] Support pagination even in loadLimit(), so that if a user
 wants to load more than 100 records but not all of them, they can. Currently,
 there's a bug where the document loader doesn't work on loading more than 100
 records because of internal Airtable API limitations. The Airtable API can
 only fetch up to 100 records per call - anything more than that requires
 pagination.

---
 .../documentloaders/Airtable/Airtable.ts      | 24 +++++++++++++++----
 1 file changed, 19 insertions(+), 5 deletions(-)

diff --git a/packages/components/nodes/documentloaders/Airtable/Airtable.ts b/packages/components/nodes/documentloaders/Airtable/Airtable.ts
index 76d6e349..011975a7 100644
--- a/packages/components/nodes/documentloaders/Airtable/Airtable.ts
+++ b/packages/components/nodes/documentloaders/Airtable/Airtable.ts
@@ -251,7 +251,7 @@ class AirtableLoader extends BaseDocumentLoader {
 
     private async loadLimit(): Promise<Document[]> {
         let data: AirtableLoaderRequest = {
-            maxRecords: this.limit,
+            maxRecords: Math.min(this.limit, 100), // Airtable only returns up to 100 records per request
             view: this.viewId
         }
 
@@ -259,11 +259,25 @@ class AirtableLoader extends BaseDocumentLoader {
             data.fields = this.fields
         }
 
-        const response = await this.fetchAirtableData(`https://api.airtable.com/v0/${this.baseId}/${this.tableId}/listRecords`, data)
-        if (response.records.length === 0) {
-            return []
+        let response: AirtableLoaderResponse
+        let returnPages: AirtableLoaderPage[] = []
+
+        // Paginate if the user specifies a limit > 100 (like 200) but not return all.
+        do {
+            response = await this.fetchAirtableData(`https://api.airtable.com/v0/${this.baseId}/${this.tableId}/listRecords`, data)
+            returnPages.push(...response.records)
+            data.offset = response.offset
+
+            // Stop if we have fetched enough records
+            if (returnPages.length >= this.limit) break
+        } while (response.offset !== undefined)
+
+        // Truncate array to the limit if necessary
+        if (returnPages.length > this.limit) {
+            returnPages.length = this.limit
         }
-        return response.records.map((page) => this.createDocumentFromPage(page))
+
+        return returnPages.map((page) => this.createDocumentFromPage(page))
     }
 
     private async loadAll(): Promise<Document[]> {