mirror of
https://github.com/farcasclaudiu/openclaw.git
synced 2026-06-29 07:01:40 +03:00
refactor(memory): share sync indexing helper
This commit is contained in:
@@ -0,0 +1,39 @@
|
|||||||
|
import type { DatabaseSync } from "node:sqlite";
|
||||||
|
|
||||||
|
type SyncProgress = {
|
||||||
|
completed: number;
|
||||||
|
total: number;
|
||||||
|
report: (update: { completed: number; total: number; label?: string }) => void;
|
||||||
|
};
|
||||||
|
|
||||||
|
function tickProgress(progress: SyncProgress | undefined): void {
|
||||||
|
if (!progress) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
progress.completed += 1;
|
||||||
|
progress.report({
|
||||||
|
completed: progress.completed,
|
||||||
|
total: progress.total,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function indexFileEntryIfChanged<
|
||||||
|
TEntry extends { path: string; hash: string },
|
||||||
|
>(params: {
|
||||||
|
db: DatabaseSync;
|
||||||
|
source: string;
|
||||||
|
needsFullReindex: boolean;
|
||||||
|
entry: TEntry;
|
||||||
|
indexFile: (entry: TEntry) => Promise<void>;
|
||||||
|
progress?: SyncProgress;
|
||||||
|
}): Promise<void> {
|
||||||
|
const record = params.db
|
||||||
|
.prepare(`SELECT hash FROM files WHERE path = ? AND source = ?`)
|
||||||
|
.get(params.entry.path, params.source) as { hash: string } | undefined;
|
||||||
|
if (!params.needsFullReindex && record?.hash === params.entry.hash) {
|
||||||
|
tickProgress(params.progress);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
await params.indexFile(params.entry);
|
||||||
|
tickProgress(params.progress);
|
||||||
|
}
|
||||||
@@ -1,6 +1,7 @@
|
|||||||
import type { DatabaseSync } from "node:sqlite";
|
import type { DatabaseSync } from "node:sqlite";
|
||||||
import { createSubsystemLogger } from "../logging/subsystem.js";
|
import { createSubsystemLogger } from "../logging/subsystem.js";
|
||||||
import { buildFileEntry, listMemoryFiles, type MemoryFileEntry } from "./internal.js";
|
import { buildFileEntry, listMemoryFiles, type MemoryFileEntry } from "./internal.js";
|
||||||
|
import { indexFileEntryIfChanged } from "./sync-index.js";
|
||||||
import { deleteStaleIndexedPaths } from "./sync-stale.js";
|
import { deleteStaleIndexedPaths } from "./sync-stale.js";
|
||||||
|
|
||||||
const log = createSubsystemLogger("memory");
|
const log = createSubsystemLogger("memory");
|
||||||
@@ -51,27 +52,14 @@ export async function syncMemoryFiles(params: {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const tasks = fileEntries.map((entry) => async () => {
|
const tasks = fileEntries.map((entry) => async () => {
|
||||||
const record = params.db
|
await indexFileEntryIfChanged({
|
||||||
.prepare(`SELECT hash FROM files WHERE path = ? AND source = ?`)
|
db: params.db,
|
||||||
.get(entry.path, "memory") as { hash: string } | undefined;
|
source: "memory",
|
||||||
if (!params.needsFullReindex && record?.hash === entry.hash) {
|
needsFullReindex: params.needsFullReindex,
|
||||||
if (params.progress) {
|
entry,
|
||||||
params.progress.completed += 1;
|
indexFile: params.indexFile,
|
||||||
params.progress.report({
|
progress: params.progress,
|
||||||
completed: params.progress.completed,
|
|
||||||
total: params.progress.total,
|
|
||||||
});
|
});
|
||||||
}
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
await params.indexFile(entry);
|
|
||||||
if (params.progress) {
|
|
||||||
params.progress.completed += 1;
|
|
||||||
params.progress.report({
|
|
||||||
completed: params.progress.completed,
|
|
||||||
total: params.progress.total,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
});
|
});
|
||||||
|
|
||||||
await params.runWithConcurrency(tasks, params.concurrency);
|
await params.runWithConcurrency(tasks, params.concurrency);
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import {
|
|||||||
listSessionFilesForAgent,
|
listSessionFilesForAgent,
|
||||||
sessionPathForFile,
|
sessionPathForFile,
|
||||||
} from "./session-files.js";
|
} from "./session-files.js";
|
||||||
|
import { indexFileEntryIfChanged } from "./sync-index.js";
|
||||||
import { deleteStaleIndexedPaths } from "./sync-stale.js";
|
import { deleteStaleIndexedPaths } from "./sync-stale.js";
|
||||||
|
|
||||||
const log = createSubsystemLogger("memory");
|
const log = createSubsystemLogger("memory");
|
||||||
@@ -76,27 +77,14 @@ export async function syncSessionFiles(params: {
|
|||||||
}
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const record = params.db
|
await indexFileEntryIfChanged({
|
||||||
.prepare(`SELECT hash FROM files WHERE path = ? AND source = ?`)
|
db: params.db,
|
||||||
.get(entry.path, "sessions") as { hash: string } | undefined;
|
source: "sessions",
|
||||||
if (!params.needsFullReindex && record?.hash === entry.hash) {
|
needsFullReindex: params.needsFullReindex,
|
||||||
if (params.progress) {
|
entry,
|
||||||
params.progress.completed += 1;
|
indexFile: params.indexFile,
|
||||||
params.progress.report({
|
progress: params.progress,
|
||||||
completed: params.progress.completed,
|
|
||||||
total: params.progress.total,
|
|
||||||
});
|
});
|
||||||
}
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
await params.indexFile(entry);
|
|
||||||
if (params.progress) {
|
|
||||||
params.progress.completed += 1;
|
|
||||||
params.progress.report({
|
|
||||||
completed: params.progress.completed,
|
|
||||||
total: params.progress.total,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
});
|
});
|
||||||
|
|
||||||
await params.runWithConcurrency(tasks, params.concurrency);
|
await params.runWithConcurrency(tasks, params.concurrency);
|
||||||
|
|||||||
Reference in New Issue
Block a user