1099 lines
46 KiB
JavaScript
1099 lines
46 KiB
JavaScript
"use strict";
|
||
|
||
/*
|
||
* LINE WORKS 掲示板 → AnythingLLM RAG 連携スクリプト (v3)
|
||
*
|
||
* getGroupList.js の認証方式(crypto標準モジュールによるJWT自前署名)を流用。
|
||
* Node.js 18+ の標準 fetch / FormData / Blob のみを使用するため、
|
||
* 追加の npm install は不要です。
|
||
*
|
||
* v3: 投稿本文に加え、添付ファイル(PDF・画像等)もダウンロードして
|
||
* AnythingLLMに個別アップロードする(PDF/画像はAnythingLLM側でOCR/テキスト抽出)。
|
||
*
|
||
* 仕様書: https://developers.worksmobile.com/jp/docs/board
|
||
*
|
||
* 使い方:
|
||
* node lineworks-anythingllm.js sync
|
||
* node lineworks-anythingllm.js list
|
||
* node lineworks-anythingllm.js ask "問い合わせ内容"
|
||
*
|
||
* 環境変数 (getGroupList.js と共通の命名):
|
||
* LW_CLIENT_ID
|
||
* LW_CLIENT_SECRET
|
||
* LW_SERVICE_ACCOUNT
|
||
* LW_PRIVATE_KEY (鍵の中身を直接渡す場合)
|
||
* LW_PRIVATE_KEY_FILE (鍵ファイルのパスを渡す場合)
|
||
* LW_SCOPE 既定: board
|
||
*
|
||
* board-list.csv (このスクリプトと同じフォルダに配置):
|
||
* boardId,boardName,flag の形式。flag列が "1" の行だけが同期対象になる。
|
||
*
|
||
* ANYTHINGLLM_BASE_URL 例: http://localhost:3001
|
||
* ANYTHINGLLM_API_KEY
|
||
* ANYTHINGLLM_WORKSPACE_SLUG 例: lineworks-board
|
||
*
|
||
* 取り込み完了ファイルの退避: AnythingLLMへのアップロードが成功した投稿Markdown/
|
||
* 添付ファイルは、それぞれ board_posts_md/complete/ ・ board_attachments/complete/
|
||
* へ移動する(次回syncでの重複アップロードを避けるため)。
|
||
* 同名ファイルが既にcomplete側にある場合は上書きせず、タイムスタンプを付けて退避する。
|
||
*
|
||
* 重複防止: board-uploaded-hashes.json (このスクリプトと同じフォルダ) に、
|
||
* アップロード済みの投稿・添付ファイルのハッシュを記録し、次回以降のsyncで
|
||
* 同一内容の再アップロードをスキップする。
|
||
* - 投稿本文: boardId+postId+更新日時+タイトル+本文からハッシュ化。
|
||
* 本文が編集され更新日時が進むと別ハッシュになり、再アップロードされる。
|
||
* - 添付ファイル: boardId+postId+attachmentId(不変)からハッシュ化。
|
||
*/
|
||
|
||
const fs = require("fs");
|
||
const path = require("path");
|
||
const crypto = require("crypto");
|
||
|
||
// ====================================================================
|
||
// Rate Limit対策: スロットリング + 429時の自動リトライ
|
||
// ====================================================================
|
||
|
||
// 各API呼び出しの間隔(ミリ秒)。既定250ms = 理論上 最大240req/分相当まで抑制。
|
||
const LW_REQUEST_DELAY_MS = Number(process.env.LW_REQUEST_DELAY_MS || 250);
|
||
// 429発生時の最大リトライ回数
|
||
const LW_MAX_RETRIES = Number(process.env.LW_MAX_RETRIES || 5);
|
||
|
||
// JSON.parseの数値型(IEEE754)では19桁のboardId/postIdが精度落ちして壊れるため、
|
||
// 指定キーの整数値を文字列化してからparseする (importBoard.js から移植)
|
||
// 例: "boardId": 4080000000758656500 -> "boardId":"4080000000758656500"
|
||
function parseJsonWithQuotedInt64(text, keys) {
|
||
let patched = String(text || "");
|
||
for (const key of keys) {
|
||
const re = new RegExp(`"${key}"\\s*:\\s*(\\d+)`, "g");
|
||
patched = patched.replace(re, `"${key}":"$1"`);
|
||
}
|
||
return JSON.parse(patched);
|
||
}
|
||
|
||
// board-list.csv (boardId,boardName,flag) を読み込み、flag列が "1" の行だけを取得対象とする
|
||
const BOARD_LIST_CSV_PATH = path.join(__dirname, "board-list.csv");
|
||
|
||
function loadTargetBoardsFromCsv(csvPath = BOARD_LIST_CSV_PATH) {
|
||
if (!fs.existsSync(csvPath)) {
|
||
return [];
|
||
}
|
||
const lines = fs
|
||
.readFileSync(csvPath, "utf8")
|
||
.split(/\r?\n/)
|
||
.map((line) => line.trim())
|
||
.filter(Boolean);
|
||
|
||
// 先頭行は "boardId,boardName" のヘッダーなのでスキップ
|
||
return lines
|
||
.slice(1)
|
||
.map((line) => line.split(",").map((s) => s.trim()))
|
||
.filter(([, , flag]) => flag === "1")
|
||
.map(([boardId, boardName]) => ({ boardId, boardName }));
|
||
}
|
||
|
||
function sleep(ms) {
|
||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||
}
|
||
|
||
// LINE WORKS API呼び出し専用のfetchラッパー。
|
||
// - 呼び出し前に一定間隔のウェイトを入れる(スロットリング)
|
||
// - 429 (Rate Limit) の場合はRetry-Afterヘッダ、または指数バックオフで自動リトライ
|
||
async function lwFetch(url, options = {}) {
|
||
for (let attempt = 0; attempt <= LW_MAX_RETRIES; attempt++) {
|
||
await sleep(LW_REQUEST_DELAY_MS);
|
||
|
||
const response = await fetch(url, options);
|
||
|
||
if (response.status !== 429) {
|
||
return response;
|
||
}
|
||
|
||
if (attempt === LW_MAX_RETRIES) {
|
||
return response; // リトライ上限に達したら、429のレスポンスをそのまま返す(呼び出し元でエラー処理)
|
||
}
|
||
|
||
const retryAfterHeader = response.headers.get("retry-after");
|
||
const retryAfterMs = retryAfterHeader
|
||
? Number(retryAfterHeader) * 1000
|
||
: LW_REQUEST_DELAY_MS * Math.pow(2, attempt + 1); // 指数バックオフ
|
||
|
||
console.warn(` [Rate Limit] 429を検知。${Math.round(retryAfterMs / 1000)}秒待機してリトライします... (${attempt + 1}/${LW_MAX_RETRIES})`);
|
||
await sleep(retryAfterMs);
|
||
}
|
||
}
|
||
|
||
|
||
|
||
const LW_TOKEN_URL = "https://auth.worksmobile.com/oauth2/v2.0/token";
|
||
const LW_API_BASE_URL = "https://www.worksapis.com/v1.0";
|
||
|
||
// getGroupList.js と同じフォルダ配置を想定。別フォルダに置く場合は
|
||
// LW_PRIVATE_KEY_FILE 環境変数で鍵ファイルの場所を明示してください。
|
||
const PRIVATE_KEY_FILE = process.env.LW_PRIVATE_KEY_FILE
|
||
? path.resolve(process.env.LW_PRIVATE_KEY_FILE)
|
||
: path.join(__dirname, "private_20260307184804.key");
|
||
|
||
const LW_CLIENT_ID = process.env.LW_CLIENT_ID || "tre8J_Tk8RblfsMSyZRh";
|
||
const LW_CLIENT_SECRET = process.env.LW_CLIENT_SECRET || "O5_R5gcHxg";
|
||
const LW_SERVICE_ACCOUNT = process.env.LW_SERVICE_ACCOUNT || "wh4k8.serviceaccount@nexthd.jp";
|
||
// 掲示板APIには board スコープが必要 (group.read ではアクセス不可)
|
||
// 投稿の読み取りには board.read も別途必要な場合があるため両方指定
|
||
const LW_SCOPE = process.env.LW_SCOPE || "board board.read";
|
||
const LW_BOARD_LIST_COUNT_PER_PAGE = 100;
|
||
const LW_POST_LIST_COUNT_PER_PAGE = 40; // 投稿一覧APIのcount上限は40
|
||
|
||
const LW_PRIVATE_KEY = (process.env.LW_PRIVATE_KEY && process.env.LW_PRIVATE_KEY.trim())
|
||
? process.env.LW_PRIVATE_KEY
|
||
: (fs.existsSync(PRIVATE_KEY_FILE) ? fs.readFileSync(PRIVATE_KEY_FILE, "utf8") : "");
|
||
|
||
function ensureRequiredEnvForAuth() {
|
||
const missing = [];
|
||
if (!LW_CLIENT_ID) missing.push("LW_CLIENT_ID");
|
||
if (!LW_CLIENT_SECRET) missing.push("LW_CLIENT_SECRET");
|
||
if (!LW_SERVICE_ACCOUNT) missing.push("LW_SERVICE_ACCOUNT");
|
||
if (!LW_PRIVATE_KEY) missing.push("LW_PRIVATE_KEY");
|
||
|
||
if (missing.length > 0) {
|
||
if (missing.includes("LW_PRIVATE_KEY") && !fs.existsSync(PRIVATE_KEY_FILE)) {
|
||
throw new Error(`秘密鍵が見つかりません: ${PRIVATE_KEY_FILE}`);
|
||
}
|
||
throw new Error(`環境変数が不足しています: ${missing.join(", ")}`);
|
||
}
|
||
}
|
||
|
||
function base64UrlEncode(value) {
|
||
return Buffer.from(value)
|
||
.toString("base64")
|
||
.replace(/=/g, "")
|
||
.replace(/\+/g, "-")
|
||
.replace(/\//g, "_");
|
||
}
|
||
|
||
function createJwtAssertion() {
|
||
const now = Math.floor(Date.now() / 1000);
|
||
const header = { alg: "RS256", typ: "JWT" };
|
||
const payload = {
|
||
iss: LW_CLIENT_ID,
|
||
sub: LW_SERVICE_ACCOUNT,
|
||
aud: LW_TOKEN_URL,
|
||
iat: now,
|
||
exp: now + 300,
|
||
};
|
||
|
||
const encodedHeader = base64UrlEncode(JSON.stringify(header));
|
||
const encodedPayload = base64UrlEncode(JSON.stringify(payload));
|
||
const signingInput = `${encodedHeader}.${encodedPayload}`;
|
||
|
||
const signer = crypto.createSign("RSA-SHA256");
|
||
signer.update(signingInput);
|
||
signer.end();
|
||
|
||
const signature = signer
|
||
.sign(LW_PRIVATE_KEY)
|
||
.toString("base64")
|
||
.replace(/=/g, "")
|
||
.replace(/\+/g, "-")
|
||
.replace(/\//g, "_");
|
||
|
||
return `${signingInput}.${signature}`;
|
||
}
|
||
|
||
async function getAccessToken(scope = LW_SCOPE) {
|
||
ensureRequiredEnvForAuth();
|
||
|
||
const assertion = createJwtAssertion();
|
||
const form = new URLSearchParams({
|
||
grant_type: "urn:ietf:params:oauth:grant-type:jwt-bearer",
|
||
assertion,
|
||
client_id: LW_CLIENT_ID,
|
||
client_secret: LW_CLIENT_SECRET,
|
||
scope,
|
||
});
|
||
|
||
const response = await lwFetch(LW_TOKEN_URL, {
|
||
method: "POST",
|
||
headers: { "Content-Type": "application/x-www-form-urlencoded" },
|
||
body: form,
|
||
});
|
||
|
||
if (!response.ok) {
|
||
const detail = await response.text();
|
||
throw new Error(`アクセストークン取得失敗: ${response.status} ${detail}`);
|
||
}
|
||
|
||
const data = await response.json();
|
||
if (!data.access_token) {
|
||
throw new Error("アクセストークン取得失敗: access_token が返却されませんでした");
|
||
}
|
||
|
||
return data.access_token;
|
||
}
|
||
|
||
// ====================================================================
|
||
// トークン自動更新 (アクセストークンの有効期限は1時間。長時間のsyncで
|
||
// 途中失効し401になるのを防ぐため、45分経過ごとに自動再取得する)
|
||
// ====================================================================
|
||
|
||
const TOKEN_REFRESH_INTERVAL_MS = 45 * 60 * 1000; // 45分
|
||
|
||
function createAuthState(initialToken) {
|
||
return { token: initialToken, obtainedAt: Date.now() };
|
||
}
|
||
|
||
async function ensureFreshToken(authState) {
|
||
const elapsed = Date.now() - authState.obtainedAt;
|
||
if (elapsed > TOKEN_REFRESH_INTERVAL_MS) {
|
||
console.log(" [トークン更新] 経過時間が長いためアクセストークンを再取得します...");
|
||
authState.token = await getAccessToken();
|
||
authState.obtainedAt = Date.now();
|
||
}
|
||
return authState.token;
|
||
}
|
||
|
||
// ====================================================================
|
||
// Board API
|
||
// ====================================================================
|
||
|
||
// アクセス可能な全掲示板を取得 (role=READERで読み取り権限のある掲示板全件)
|
||
async function fetchAllBoards(accessToken) {
|
||
const boards = [];
|
||
let cursor = "";
|
||
|
||
while (true) {
|
||
const params = new URLSearchParams();
|
||
params.set("role", "READER");
|
||
params.set("count", String(LW_BOARD_LIST_COUNT_PER_PAGE));
|
||
if (cursor) params.set("cursor", cursor);
|
||
|
||
const url = `${LW_API_BASE_URL}/boards?${params.toString()}`;
|
||
const response = await lwFetch(url, {
|
||
headers: { Authorization: `Bearer ${accessToken}` },
|
||
});
|
||
|
||
const raw = await response.text();
|
||
let data;
|
||
try {
|
||
data = raw ? parseJsonWithQuotedInt64(raw, ["boardId"]) : {};
|
||
} catch (_err) {
|
||
throw new Error(`掲示板一覧レスポンスがJSONではありません: ${raw}`);
|
||
}
|
||
|
||
if (!response.ok) {
|
||
throw new Error(`掲示板一覧取得失敗: ${response.status} ${JSON.stringify(data)}`);
|
||
}
|
||
|
||
const pageBoards = Array.isArray(data.boards) ? data.boards : [];
|
||
boards.push(...pageBoards);
|
||
|
||
const nextCursor = data?.responseMetaData?.nextCursor || "";
|
||
if (!nextCursor || nextCursor === cursor) break;
|
||
cursor = nextCursor;
|
||
}
|
||
|
||
return boards;
|
||
}
|
||
|
||
// 動作確認用: 最新投稿リストの取得 (掲示板を指定せず、直近30日の新着投稿を横断取得)
|
||
// 権限モデルが個別のboard指定エンドポイントと異なる可能性があるため、切り分け用に用意
|
||
async function fetchRecentPosts(accessToken) {
|
||
const url = `${LW_API_BASE_URL}/boards/posts?count=40`;
|
||
const response = await lwFetch(url, {
|
||
headers: { Authorization: `Bearer ${accessToken}` },
|
||
});
|
||
|
||
const raw = await response.text();
|
||
let data;
|
||
try {
|
||
data = raw ? parseJsonWithQuotedInt64(raw, ["boardId", "postId"]) : {};
|
||
} catch (_err) {
|
||
throw new Error(`最新投稿レスポンスがJSONではありません: ${raw}`);
|
||
}
|
||
|
||
if (!response.ok) {
|
||
throw new Error(`最新投稿取得失敗: ${response.status} ${JSON.stringify(data)}`);
|
||
}
|
||
|
||
return data.posts || [];
|
||
}
|
||
|
||
async function fetchAllPosts(accessToken, boardId) {
|
||
const posts = [];
|
||
let cursor = "";
|
||
|
||
while (true) {
|
||
const params = new URLSearchParams();
|
||
params.set("count", String(LW_POST_LIST_COUNT_PER_PAGE));
|
||
if (cursor) params.set("cursor", cursor);
|
||
|
||
const url = `${LW_API_BASE_URL}/boards/${boardId}/posts?${params.toString()}`;
|
||
const response = await lwFetch(url, {
|
||
headers: { Authorization: `Bearer ${accessToken}` },
|
||
});
|
||
|
||
const raw = await response.text();
|
||
let data;
|
||
try {
|
||
data = raw ? parseJsonWithQuotedInt64(raw, ["boardId", "postId"]) : {};
|
||
} catch (_err) {
|
||
throw new Error(`投稿一覧レスポンスがJSONではありません: ${raw}`);
|
||
}
|
||
|
||
if (!response.ok) {
|
||
throw new Error(`投稿一覧取得失敗 (board ${boardId}): ${response.status} ${JSON.stringify(data)}`);
|
||
}
|
||
|
||
const pagePosts = Array.isArray(data.posts) ? data.posts : [];
|
||
posts.push(...pagePosts);
|
||
|
||
const nextCursor = data?.responseMetaData?.nextCursor || "";
|
||
if (!nextCursor || nextCursor === cursor) break;
|
||
cursor = nextCursor;
|
||
}
|
||
|
||
return posts;
|
||
}
|
||
|
||
async function fetchPostDetail(accessToken, boardId, postId) {
|
||
const url = `${LW_API_BASE_URL}/boards/${boardId}/posts/${postId}`;
|
||
const response = await lwFetch(url, {
|
||
headers: { Authorization: `Bearer ${accessToken}` },
|
||
});
|
||
|
||
const raw = await response.text();
|
||
let data;
|
||
try {
|
||
data = raw ? parseJsonWithQuotedInt64(raw, ["boardId", "postId"]) : {};
|
||
} catch (_err) {
|
||
throw new Error(`投稿詳細レスポンスがJSONではありません: ${raw}`);
|
||
}
|
||
|
||
if (!response.ok) {
|
||
throw new Error(`投稿詳細取得失敗 (post ${postId}): ${response.status} ${JSON.stringify(data)}`);
|
||
}
|
||
|
||
return data;
|
||
}
|
||
|
||
// 添付ファイル一覧を取得 (PDF・画像等、fileCount > 0の投稿にのみ存在)
|
||
async function fetchPostAttachments(accessToken, boardId, postId) {
|
||
const attachments = [];
|
||
let cursor = "";
|
||
|
||
while (true) {
|
||
const params = new URLSearchParams();
|
||
params.set("count", "20"); // 仕様上の最大値
|
||
if (cursor) params.set("cursor", cursor);
|
||
|
||
const url = `${LW_API_BASE_URL}/boards/${boardId}/posts/${postId}/attachments?${params.toString()}`;
|
||
const response = await lwFetch(url, {
|
||
headers: { Authorization: `Bearer ${accessToken}` },
|
||
});
|
||
|
||
const raw = await response.text();
|
||
let data;
|
||
try {
|
||
data = raw ? JSON.parse(raw) : {}; // attachmentIdは既にstring型なので精度落ちの心配なし
|
||
} catch (_err) {
|
||
throw new Error(`添付ファイル一覧レスポンスがJSONではありません: ${raw}`);
|
||
}
|
||
|
||
if (!response.ok) {
|
||
throw new Error(`添付ファイル一覧取得失敗 (post ${postId}): ${response.status} ${JSON.stringify(data)}`);
|
||
}
|
||
|
||
const pageAttachments = Array.isArray(data.attachments) ? data.attachments : [];
|
||
attachments.push(...pageAttachments);
|
||
|
||
const nextCursor = data?.responseMetaData?.nextCursor || "";
|
||
if (!nextCursor || nextCursor === cursor) break;
|
||
cursor = nextCursor;
|
||
}
|
||
|
||
return attachments;
|
||
}
|
||
|
||
// 添付ファイル本体をダウンロード (302リダイレクト先から取得。fetchは既定でリダイレクトを自動追跡する)
|
||
async function downloadAttachment(accessToken, boardId, postId, attachmentId) {
|
||
const url = `${LW_API_BASE_URL}/boards/${boardId}/posts/${postId}/attachments/${attachmentId}`;
|
||
|
||
// 重要: fetchは既定で別ドメインへのリダイレクト時にAuthorizationヘッダーを
|
||
// 自動的に取り除く。しかしLINE WORKSのリダイレクト先(apis-storage.worksmobile.com)は
|
||
// 同じBearerトークンでの認証を要求するため、リダイレクトを手動で処理し
|
||
// 自前でAuthorizationヘッダーを引き継ぐ必要がある。
|
||
let response = await lwFetch(url, {
|
||
headers: { Authorization: `Bearer ${accessToken}` },
|
||
redirect: "manual",
|
||
});
|
||
|
||
if (response.status >= 300 && response.status < 400) {
|
||
const location = response.headers.get("location");
|
||
if (!location) {
|
||
throw new Error(`添付ファイルダウンロード失敗 (attachment ${attachmentId}): リダイレクト先が取得できません`);
|
||
}
|
||
response = await lwFetch(location, {
|
||
headers: { Authorization: `Bearer ${accessToken}` },
|
||
});
|
||
}
|
||
|
||
if (!response.ok) {
|
||
let bodyText = "";
|
||
try {
|
||
bodyText = await response.text();
|
||
} catch (_err) {
|
||
bodyText = "(本文取得失敗)";
|
||
}
|
||
throw new Error(
|
||
`添付ファイルダウンロード失敗 (attachment ${attachmentId}): ${response.status} ` +
|
||
`finalUrl=${response.url} body=${bodyText}`
|
||
);
|
||
}
|
||
|
||
const arrayBuffer = await response.arrayBuffer();
|
||
return Buffer.from(arrayBuffer);
|
||
}
|
||
|
||
// ====================================================================
|
||
// 投稿をMarkdownファイル化 / 添付ファイルの保存
|
||
// ====================================================================
|
||
|
||
const EXPORT_DIR = path.join(__dirname, "board_posts_md");
|
||
const EXPORT_COMPLETE_DIR = path.join(EXPORT_DIR, "complete");
|
||
const ATTACHMENT_DIR = path.join(__dirname, "board_attachments");
|
||
const ATTACHMENT_COMPLETE_DIR = path.join(ATTACHMENT_DIR, "complete");
|
||
|
||
// AnythingLLMへのアップロードが完了したファイルを、元のディレクトリ配下の
|
||
// complete/ サブフォルダへ移動する(次回syncで再アップロード対象に含めないため)。
|
||
// 同名ファイルが既にcomplete側にある場合は上書きせず、タイムスタンプを付けて退避する。
|
||
function moveFileToCompleteDir(srcPath, completeDir) {
|
||
fs.mkdirSync(completeDir, { recursive: true });
|
||
const filename = path.basename(srcPath);
|
||
const ext = path.extname(filename);
|
||
const base = path.basename(filename, ext);
|
||
let destPath = path.join(completeDir, filename);
|
||
if (fs.existsSync(destPath)) {
|
||
destPath = path.join(completeDir, `${base}_${timestampForFileName()}${ext}`);
|
||
}
|
||
fs.renameSync(srcPath, destPath);
|
||
return destPath;
|
||
}
|
||
|
||
// ====================================================================
|
||
// 重複防止 (投稿本文・添付ファイルの再アップロードをスキップ)
|
||
// ====================================================================
|
||
//
|
||
// - 投稿本文: boardId+postId+更新日時+タイトル+本文からハッシュを生成する。
|
||
// 本文が編集され更新日時が進むとハッシュが変わり、再アップロード対象になる。
|
||
// - 添付ファイル: attachmentIdはLINE WORKS側で不変のため、boardId+postId+attachmentId
|
||
// のみでハッシュ化する(内容の再ダウンロード・再アップロードを避けるため)。
|
||
// 記録先: board-uploaded-hashes.json (このスクリプトと同じフォルダ)
|
||
|
||
const UPLOADED_HASHES_PATH = path.join(__dirname, "board-uploaded-hashes.json");
|
||
|
||
function contentHash(...parts) {
|
||
return crypto.createHash("sha256").update(parts.join("|")).digest("hex");
|
||
}
|
||
|
||
function postContentHash(boardId, postId, detail, bodyText) {
|
||
return contentHash("post", boardId, postId, detail.updatedTime || detail.createdTime || "", detail.title || "", bodyText);
|
||
}
|
||
|
||
function attachmentIdentityHash(boardId, postId, attachment) {
|
||
return contentHash("attachment", boardId, postId, attachment.attachmentId);
|
||
}
|
||
|
||
function loadUploadedHashes() {
|
||
if (!fs.existsSync(UPLOADED_HASHES_PATH)) {
|
||
return new Set();
|
||
}
|
||
try {
|
||
const arr = JSON.parse(fs.readFileSync(UPLOADED_HASHES_PATH, "utf8"));
|
||
return new Set(Array.isArray(arr) ? arr : []);
|
||
} catch {
|
||
return new Set();
|
||
}
|
||
}
|
||
|
||
function saveUploadedHashes(hashSet) {
|
||
fs.writeFileSync(UPLOADED_HASHES_PATH, JSON.stringify([...hashSet]), "utf8");
|
||
}
|
||
|
||
function stripHtml(html) {
|
||
return (html || "").replace(/<[^>]+>/g, "").trim();
|
||
}
|
||
|
||
// ファイル名として安全な形に変換 (パス区切り文字等を除去)
|
||
function sanitizeFileName(name) {
|
||
return String(name || "file").replace(/[\\/:*?"<>|]/g, "_");
|
||
}
|
||
|
||
function writePostAsMarkdown(boardId, post, detail, attachments = []) {
|
||
fs.mkdirSync(EXPORT_DIR, { recursive: true });
|
||
const filename = `board${boardId}_post${post.postId}.md`;
|
||
const filepath = path.join(EXPORT_DIR, filename);
|
||
|
||
const lines = [
|
||
`# ${detail.title || "(無題)"}`,
|
||
"",
|
||
`- 掲示板ID: ${boardId}`,
|
||
`- 投稿ID: ${post.postId}`,
|
||
`- 投稿日時: ${detail.createdTime || ""}`,
|
||
`- 投稿者: ${detail.userName || detail.userId || ""}`,
|
||
];
|
||
|
||
if (attachments.length > 0) {
|
||
lines.push(`- 添付ファイル: ${attachments.map((a) => a.fileName).join(", ")}`);
|
||
}
|
||
|
||
lines.push("", stripHtml(detail.body));
|
||
|
||
fs.writeFileSync(filepath, lines.join("\n"), "utf8");
|
||
return filepath;
|
||
}
|
||
|
||
// ====================================================================
|
||
// 規則集(章・条形式)の分割 — 正規表現による確実な分割 (LLM不使用)
|
||
// ====================================================================
|
||
|
||
// 「第◯条」を条区切りとして検出。見出しが()内にある場合はそれも取得する。
|
||
// 例: "第76条（特別休暇）" "第94条(事業引継)" "第123条"
|
||
const ARTICLE_PATTERN = /第([0-9０-９]+)条\s*[（(]?([^）)\n第]{0,30})?[）)]?/g;
|
||
|
||
// 分割対象かどうかを判定する閾値。短い投稿や条文形式でない投稿は対象外。
|
||
const ARTICLE_SPLIT_MIN_LENGTH = 3000;
|
||
const ARTICLE_SPLIT_MIN_COUNT = 3;
|
||
|
||
function splitIntoArticles(bodyText) {
|
||
const matches = [...bodyText.matchAll(ARTICLE_PATTERN)];
|
||
|
||
if (bodyText.length < ARTICLE_SPLIT_MIN_LENGTH || matches.length < ARTICLE_SPLIT_MIN_COUNT) {
|
||
return null; // 分割対象外(通常の投稿として扱う)
|
||
}
|
||
|
||
const articles = [];
|
||
|
||
// 最初の条文より前にある前文(総則等)を1つの塊として保持
|
||
const firstIndex = matches[0].index;
|
||
if (firstIndex > 50) {
|
||
const preamble = bodyText.slice(0, firstIndex).trim();
|
||
if (preamble) {
|
||
articles.push({ articleNumber: "前文", heading: "", text: preamble });
|
||
}
|
||
}
|
||
|
||
for (let i = 0; i < matches.length; i++) {
|
||
const start = matches[i].index;
|
||
const end = i + 1 < matches.length ? matches[i + 1].index : bodyText.length;
|
||
const articleNumber = matches[i][1];
|
||
const heading = (matches[i][2] || "").trim();
|
||
const text = bodyText.slice(start, end).trim();
|
||
articles.push({ articleNumber, heading, text });
|
||
}
|
||
|
||
return articles;
|
||
}
|
||
|
||
// 条ごとに個別のMarkdownファイルとして出力
|
||
function writeArticlesAsMarkdown(boardId, post, detail, articles) {
|
||
fs.mkdirSync(EXPORT_DIR, { recursive: true });
|
||
const filepaths = [];
|
||
|
||
for (const article of articles) {
|
||
const label = article.articleNumber === "前文" ? "前文" : `第${article.articleNumber}条`;
|
||
const headingSuffix = article.heading ? `（${article.heading}）` : "";
|
||
const filename = `board${boardId}_post${post.postId}_art${sanitizeFileName(article.articleNumber)}.md`;
|
||
const filepath = path.join(EXPORT_DIR, filename);
|
||
|
||
const lines = [
|
||
`# ${detail.title || "(無題)"} - ${label}${headingSuffix}`,
|
||
"",
|
||
`- 掲示板ID: ${boardId}`,
|
||
`- 投稿ID: ${post.postId}`,
|
||
`- 元の投稿タイトル: ${detail.title || ""}`,
|
||
`- 条番号: ${label}${headingSuffix}`,
|
||
`- 投稿日時: ${detail.createdTime || ""}`,
|
||
"",
|
||
article.text,
|
||
];
|
||
|
||
fs.writeFileSync(filepath, lines.join("\n"), "utf8");
|
||
filepaths.push(filepath);
|
||
}
|
||
|
||
return filepaths;
|
||
}
|
||
|
||
// ====================================================================
|
||
// 条文形式でない長文投稿のAI分割 — LM Studioに「区切り位置」だけ判断させ、
|
||
// 分割自体はプログラム側で機械的に行う(本文の書き換え・要約はさせない)
|
||
// ====================================================================
|
||
|
||
const AI_SPLIT_MIN_LENGTH = Number(process.env.AI_SPLIT_MIN_LENGTH || 3000);
|
||
const LMSTUDIO_CHAT_URL = process.env.LMSTUDIO_CHAT_URL || "http://localhost:1234/v1/chat/completions";
|
||
const LMSTUDIO_MODEL_NAME = process.env.LMSTUDIO_MODEL_NAME || "qwen2.5-14b-instruct";
|
||
const AI_SPLIT_MARKER = "===SPLIT===";
|
||
|
||
// 分割結果が原文を改変していないかの許容誤差(文字数ベース)
|
||
const AI_SPLIT_LENGTH_TOLERANCE = 0.1; // ±10%まで許容
|
||
|
||
async function splitWithAI(bodyText) {
|
||
const prompt =
|
||
`以下の文章を、意味のまとまり(話題)ごとに分割してください。\n` +
|
||
`分割したい箇所にだけ、改行して「${AI_SPLIT_MARKER}」という文字列を挿入してください。\n` +
|
||
`【厳守事項】\n` +
|
||
`- 元の文章は一字一句変更しないでください。要約・言い換え・省略は禁止です。\n` +
|
||
`- 「${AI_SPLIT_MARKER}」以外の文字は追加しないでください。\n` +
|
||
`- 出力は、区切り記号を挿入した本文全体のみとしてください(前置きや説明文は不要です)。\n\n` +
|
||
`【本文】\n${bodyText}`;
|
||
|
||
const res = await fetch(LMSTUDIO_CHAT_URL, {
|
||
method: "POST",
|
||
headers: { "Content-Type": "application/json" },
|
||
body: JSON.stringify({
|
||
model: LMSTUDIO_MODEL_NAME,
|
||
messages: [{ role: "user", content: prompt }],
|
||
temperature: 0,
|
||
max_tokens: 8000,
|
||
}),
|
||
});
|
||
|
||
if (!res.ok) {
|
||
throw new Error(`LM Studio呼び出し失敗: ${res.status} ${await res.text()}`);
|
||
}
|
||
|
||
const data = await res.json();
|
||
const content = data.choices?.[0]?.message?.content || "";
|
||
|
||
const segments = content
|
||
.split(AI_SPLIT_MARKER)
|
||
.map((s) => s.trim())
|
||
.filter((s) => s.length > 0);
|
||
|
||
if (segments.length < 2) {
|
||
return null; // 分割されなかった(そのまま1ファイルとして扱う)
|
||
}
|
||
|
||
// 改変チェック: 分割後の合計文字数が原文から大きくズレていないか検証
|
||
const totalLength = segments.reduce((sum, s) => sum + s.length, 0);
|
||
const ratio = totalLength / bodyText.length;
|
||
if (ratio < 1 - AI_SPLIT_LENGTH_TOLERANCE || ratio > 1 + AI_SPLIT_LENGTH_TOLERANCE) {
|
||
console.warn(` [警告] AI分割で文字数が想定より変化(${Math.round(ratio * 100)}%)のため分割を破棄`);
|
||
return null;
|
||
}
|
||
|
||
return segments;
|
||
}
|
||
|
||
// AI分割の結果を、それぞれ個別のMarkdownファイルとして出力
|
||
function writeAiSegmentsAsMarkdown(boardId, post, detail, segments) {
|
||
fs.mkdirSync(EXPORT_DIR, { recursive: true });
|
||
const filepaths = [];
|
||
|
||
segments.forEach((segmentText, index) => {
|
||
const partNumber = index + 1;
|
||
const filename = `board${boardId}_post${post.postId}_part${partNumber}.md`;
|
||
const filepath = path.join(EXPORT_DIR, filename);
|
||
|
||
const lines = [
|
||
`# ${detail.title || "(無題)"} - Part${partNumber}/${segments.length}`,
|
||
"",
|
||
`- 掲示板ID: ${boardId}`,
|
||
`- 投稿ID: ${post.postId}`,
|
||
`- 元の投稿タイトル: ${detail.title || ""}`,
|
||
`- 投稿日時: ${detail.createdTime || ""}`,
|
||
"",
|
||
segmentText,
|
||
];
|
||
|
||
fs.writeFileSync(filepath, lines.join("\n"), "utf8");
|
||
filepaths.push(filepath);
|
||
});
|
||
|
||
return filepaths;
|
||
}
|
||
|
||
// 添付ファイル本体をローカルに保存
|
||
function saveAttachmentFile(boardId, postId, attachment, buffer) {
|
||
fs.mkdirSync(ATTACHMENT_DIR, { recursive: true });
|
||
const safeName = sanitizeFileName(attachment.fileName);
|
||
const filename = `board${boardId}_post${postId}_${attachment.attachmentId.replace(/[^a-zA-Z0-9._-]/g, "")}_${safeName}`;
|
||
const filepath = path.join(ATTACHMENT_DIR, filename);
|
||
fs.writeFileSync(filepath, buffer);
|
||
return filepath;
|
||
}
|
||
|
||
// 拡張子からMIMEタイプを簡易判定 (AnythingLLM側の処理分岐に必要)
|
||
function guessMimeType(filename) {
|
||
const ext = path.extname(filename).toLowerCase();
|
||
const map = {
|
||
".pdf": "application/pdf",
|
||
".png": "image/png",
|
||
".jpg": "image/jpeg",
|
||
".jpeg": "image/jpeg",
|
||
".gif": "image/gif",
|
||
".webp": "image/webp",
|
||
".txt": "text/plain",
|
||
".md": "text/markdown",
|
||
".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||
".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||
".pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation",
|
||
".csv": "text/csv",
|
||
};
|
||
return map[ext] || "application/octet-stream";
|
||
}
|
||
|
||
// ====================================================================
|
||
// AnythingLLM API (標準 fetch + FormData + Blob のみ使用)
|
||
// ====================================================================
|
||
|
||
const {
|
||
ANYTHINGLLM_BASE_URL,
|
||
ANYTHINGLLM_API_KEY,
|
||
ANYTHINGLLM_WORKSPACE_SLUG,
|
||
} = process.env;
|
||
|
||
async function uploadToAnythingLLM(filepath, mimeType = "text/markdown") {
|
||
const fileBuffer = fs.readFileSync(filepath);
|
||
const blob = new Blob([fileBuffer], { type: mimeType });
|
||
|
||
const form = new FormData();
|
||
form.append("file", blob, path.basename(filepath));
|
||
|
||
const res = await fetch(`${ANYTHINGLLM_BASE_URL}/api/v1/document/upload`, {
|
||
method: "POST",
|
||
headers: { Authorization: `Bearer ${ANYTHINGLLM_API_KEY}` },
|
||
body: form,
|
||
});
|
||
|
||
if (!res.ok) {
|
||
throw new Error(`AnythingLLMアップロード失敗: ${res.status} ${await res.text()}`);
|
||
}
|
||
const data = await res.json();
|
||
return data.documents?.[0]?.location;
|
||
}
|
||
|
||
// 一度に大量のファイルを埋め込もうとするとAnythingLLM/Ollama側でタイムアウトする
|
||
// (fetch failed) ため、一定件数ごとにバッチ分割して順番に送信する
|
||
const EMBEDDING_BATCH_SIZE = Number(process.env.EMBEDDING_BATCH_SIZE || 50);
|
||
|
||
async function addToWorkspaceEmbeddings(locations) {
|
||
const totalBatches = Math.ceil(locations.length / EMBEDDING_BATCH_SIZE);
|
||
let succeededCount = 0;
|
||
const failedBatches = [];
|
||
|
||
for (let i = 0; i < locations.length; i += EMBEDDING_BATCH_SIZE) {
|
||
const batch = locations.slice(i, i + EMBEDDING_BATCH_SIZE);
|
||
const batchNumber = Math.floor(i / EMBEDDING_BATCH_SIZE) + 1;
|
||
|
||
console.log(` 埋め込み中... バッチ ${batchNumber}/${totalBatches} (${batch.length}件)`);
|
||
|
||
try {
|
||
const res = await fetch(
|
||
`${ANYTHINGLLM_BASE_URL}/api/v1/workspace/${ANYTHINGLLM_WORKSPACE_SLUG}/update-embeddings`,
|
||
{
|
||
method: "POST",
|
||
headers: {
|
||
Authorization: `Bearer ${ANYTHINGLLM_API_KEY}`,
|
||
"Content-Type": "application/json",
|
||
},
|
||
body: JSON.stringify({ adds: batch, deletes: [] }),
|
||
}
|
||
);
|
||
if (!res.ok) {
|
||
throw new Error(`${res.status} ${await res.text()}`);
|
||
}
|
||
succeededCount += batch.length;
|
||
} catch (err) {
|
||
console.warn(` [警告] バッチ ${batchNumber} の埋め込みに失敗: ${err.message}`);
|
||
failedBatches.push({ batchNumber, count: batch.length, reason: err.message });
|
||
}
|
||
}
|
||
|
||
console.log(`埋め込み完了: 成功 ${succeededCount}/${locations.length} 件`);
|
||
if (failedBatches.length > 0) {
|
||
console.warn(`失敗したバッチ: ${failedBatches.length}件`);
|
||
failedBatches.forEach((b) => console.warn(` - バッチ${b.batchNumber} (${b.count}件): ${b.reason}`));
|
||
}
|
||
|
||
return { succeededCount, failedBatches };
|
||
}
|
||
|
||
async function askWorkspace(question) {
|
||
const res = await fetch(
|
||
`${ANYTHINGLLM_BASE_URL}/api/v1/workspace/${ANYTHINGLLM_WORKSPACE_SLUG}/chat`,
|
||
{
|
||
method: "POST",
|
||
headers: {
|
||
Authorization: `Bearer ${ANYTHINGLLM_API_KEY}`,
|
||
"Content-Type": "application/json",
|
||
},
|
||
body: JSON.stringify({ message: question, mode: "query" }),
|
||
}
|
||
);
|
||
if (!res.ok) {
|
||
throw new Error(`質問失敗: ${res.status} ${await res.text()}`);
|
||
}
|
||
return res.json();
|
||
}
|
||
|
||
// ====================================================================
|
||
// コマンド
|
||
// ====================================================================
|
||
|
||
async function syncCommand() {
|
||
if (typeof fetch !== "function") {
|
||
throw new Error("この Node.js 実行環境は fetch に未対応です。Node.js 18 以上を使用してください。");
|
||
}
|
||
|
||
const authState = createAuthState(await getAccessToken());
|
||
|
||
// board-list.csv の flag列が "1" の行を優先、該当が無ければ全掲示板を自動取得
|
||
const csvBoards = loadTargetBoardsFromCsv();
|
||
|
||
let targetBoards;
|
||
if (csvBoards.length > 0) {
|
||
console.log(`${BOARD_LIST_CSV_PATH} から対象掲示板を読み込みました: ${csvBoards.length} 件`);
|
||
csvBoards.forEach((b) => console.log(` - [${b.boardId}] ${b.boardName}`));
|
||
targetBoards = csvBoards;
|
||
} else {
|
||
console.log(`${BOARD_LIST_CSV_PATH} に取得対象(flag=1)の掲示板が無いため、アクセス可能な全掲示板を取得します...`);
|
||
targetBoards = await fetchAllBoards(authState.token);
|
||
console.log(`対象掲示板: ${targetBoards.length} 件`);
|
||
targetBoards.forEach((b) => console.log(` - [${b.boardId}] ${b.boardName}`));
|
||
}
|
||
|
||
if (targetBoards.length === 0) {
|
||
throw new Error("アクセス可能な掲示板が見つかりませんでした。Service Accountの権限を確認してください。");
|
||
}
|
||
|
||
const uploadedLocations = [];
|
||
const skippedBoards = [];
|
||
const uploadedHashes = loadUploadedHashes();
|
||
let skippedPostCount = 0;
|
||
let skippedAttachmentCount = 0;
|
||
|
||
for (const board of targetBoards) {
|
||
const boardId = board.boardId;
|
||
try {
|
||
const accessToken = await ensureFreshToken(authState);
|
||
const posts = await fetchAllPosts(accessToken, boardId);
|
||
console.log(`board ${boardId} (${board.boardName}): ${posts.length} 件`);
|
||
|
||
for (const post of posts) {
|
||
try {
|
||
const accessToken = await ensureFreshToken(authState);
|
||
const detail = await fetchPostDetail(accessToken, boardId, post.postId);
|
||
|
||
// 添付ファイルがあれば先に取得しておき、Markdownの説明欄にも反映する
|
||
let attachments = [];
|
||
if (Number(post.fileCount) > 0) {
|
||
try {
|
||
attachments = await fetchPostAttachments(accessToken, boardId, post.postId);
|
||
} catch (attachListErr) {
|
||
console.warn(` [警告] 添付ファイル一覧取得に失敗 (post ${post.postId}): ${attachListErr.message}`);
|
||
}
|
||
}
|
||
|
||
const bodyText = stripHtml(detail.body);
|
||
const postHash = postContentHash(boardId, post.postId, detail, bodyText);
|
||
|
||
if (uploadedHashes.has(postHash)) {
|
||
console.log(` [スキップ(重複)] ${detail.title}`);
|
||
skippedPostCount++;
|
||
} else {
|
||
const articles = splitIntoArticles(bodyText);
|
||
|
||
if (articles) {
|
||
// 規則集など条文形式: 条ごとに個別ファイルとしてアップロード
|
||
const filepaths = writeArticlesAsMarkdown(boardId, post, detail, articles);
|
||
console.log(` 取り込み完了(条文分割): ${detail.title} → ${filepaths.length}件に分割`);
|
||
for (const fp of filepaths) {
|
||
const location = await uploadToAnythingLLM(fp);
|
||
if (location) {
|
||
uploadedLocations.push(location);
|
||
moveFileToCompleteDir(fp, EXPORT_COMPLETE_DIR);
|
||
}
|
||
}
|
||
} else if (bodyText.length > AI_SPLIT_MIN_LENGTH) {
|
||
// 条文形式ではないが長文: AIに区切り位置だけ判断させて分割を試みる
|
||
let aiSegments = null;
|
||
try {
|
||
aiSegments = await splitWithAI(bodyText);
|
||
} catch (aiErr) {
|
||
console.warn(` [警告] AI分割に失敗、通常アップロードにフォールバック: ${aiErr.message}`);
|
||
}
|
||
|
||
if (aiSegments) {
|
||
const filepaths = writeAiSegmentsAsMarkdown(boardId, post, detail, aiSegments);
|
||
console.log(` 取り込み完了(AI分割): ${detail.title} → ${filepaths.length}件に分割`);
|
||
for (const fp of filepaths) {
|
||
const location = await uploadToAnythingLLM(fp);
|
||
if (location) {
|
||
uploadedLocations.push(location);
|
||
moveFileToCompleteDir(fp, EXPORT_COMPLETE_DIR);
|
||
}
|
||
}
|
||
} else {
|
||
// AI分割が使えなかった場合は通常通り1ファイルとして扱う
|
||
const filepath = writePostAsMarkdown(boardId, post, detail, attachments);
|
||
const location = await uploadToAnythingLLM(filepath);
|
||
if (location) {
|
||
uploadedLocations.push(location);
|
||
moveFileToCompleteDir(filepath, EXPORT_COMPLETE_DIR);
|
||
}
|
||
console.log(` 取り込み完了: ${detail.title}`);
|
||
}
|
||
} else {
|
||
// 通常の投稿: 1投稿1ファイルのまま
|
||
const filepath = writePostAsMarkdown(boardId, post, detail, attachments);
|
||
const location = await uploadToAnythingLLM(filepath);
|
||
if (location) {
|
||
uploadedLocations.push(location);
|
||
moveFileToCompleteDir(filepath, EXPORT_COMPLETE_DIR);
|
||
}
|
||
console.log(` 取り込み完了: ${detail.title}`);
|
||
}
|
||
|
||
uploadedHashes.add(postHash);
|
||
saveUploadedHashes(uploadedHashes);
|
||
}
|
||
|
||
// 添付ファイル本体をダウンロードしてAnythingLLMにも個別にアップロード
|
||
// (attachmentIdは不変のため、投稿本文の重複可否に関わらず添付単位で個別に判定する)
|
||
for (const attachment of attachments) {
|
||
try {
|
||
const attachmentHash = attachmentIdentityHash(boardId, post.postId, attachment);
|
||
if (uploadedHashes.has(attachmentHash)) {
|
||
console.log(` [スキップ(重複)] 添付ファイル ${attachment.fileName}`);
|
||
skippedAttachmentCount++;
|
||
continue;
|
||
}
|
||
|
||
const freshToken = await ensureFreshToken(authState);
|
||
const buffer = await downloadAttachment(freshToken, boardId, post.postId, attachment.attachmentId);
|
||
const attachmentPath = saveAttachmentFile(boardId, post.postId, attachment, buffer);
|
||
const mimeType = guessMimeType(attachment.fileName);
|
||
const attachmentLocation = await uploadToAnythingLLM(attachmentPath, mimeType);
|
||
if (attachmentLocation) {
|
||
uploadedLocations.push(attachmentLocation);
|
||
moveFileToCompleteDir(attachmentPath, ATTACHMENT_COMPLETE_DIR);
|
||
uploadedHashes.add(attachmentHash);
|
||
saveUploadedHashes(uploadedHashes);
|
||
}
|
||
console.log(` 添付ファイル取り込み完了: ${attachment.fileName}`);
|
||
} catch (attachErr) {
|
||
console.warn(` [スキップ] 添付ファイル ${attachment.fileName} の処理に失敗: ${attachErr.message}`);
|
||
}
|
||
}
|
||
} catch (postErr) {
|
||
console.warn(` [スキップ] 投稿 ${post.postId} の処理に失敗: ${postErr.message}`);
|
||
}
|
||
}
|
||
} catch (boardErr) {
|
||
console.warn(`[スキップ] board ${boardId} (${board.boardName}) の処理に失敗: ${boardErr.message}`);
|
||
skippedBoards.push({ boardId, boardName: board.boardName, reason: boardErr.message });
|
||
}
|
||
}
|
||
|
||
console.log(`重複スキップ: 投稿 ${skippedPostCount}件 / 添付ファイル ${skippedAttachmentCount}件`);
|
||
|
||
if (uploadedLocations.length > 0) {
|
||
console.log(`${uploadedLocations.length} 件をワークスペースに埋め込み中...`);
|
||
await addToWorkspaceEmbeddings(uploadedLocations);
|
||
}
|
||
|
||
if (skippedBoards.length > 0) {
|
||
console.log(`\n=== スキップされた掲示板 (${skippedBoards.length}件) ===`);
|
||
skippedBoards.forEach((b) => console.log(` - [${b.boardId}] ${b.boardName}: ${b.reason}`));
|
||
console.log("↑ 権限不足の可能性があります。管理画面でService Accountの閲覧権限を確認してください。");
|
||
}
|
||
|
||
console.log("\n同期完了。");
|
||
}
|
||
|
||
async function askCommand(question) {
|
||
console.log("検索・回答生成中...");
|
||
const result = await askWorkspace(question);
|
||
|
||
console.log("\n=== 回答 ===\n");
|
||
console.log(result.textResponse);
|
||
|
||
if (result.sources?.length) {
|
||
console.log("\n=== 参照した投稿 ===");
|
||
for (const s of result.sources) {
|
||
console.log(`- ${s.title || s.filename}`);
|
||
}
|
||
}
|
||
}
|
||
|
||
async function testRecentCommand() {
|
||
const accessToken = await getAccessToken();
|
||
console.log("最新投稿リストの取得(全掲示板横断)を試します...");
|
||
const posts = await fetchRecentPosts(accessToken);
|
||
console.log(`取得成功: ${posts.length} 件`);
|
||
posts.slice(0, 5).forEach((p) => {
|
||
console.log(` - [board ${p.boardId}] ${p.title}`);
|
||
});
|
||
}
|
||
|
||
// ====================================================================
|
||
// CSV出力用の簡易エスケープ (カンマ・改行・ダブルクォートを含む値に対応)
|
||
// ====================================================================
|
||
function csvEscape(value) {
|
||
const str = String(value ?? "");
|
||
if (/[",\n\r]/.test(str)) {
|
||
return `"${str.replace(/"/g, '""')}"`;
|
||
}
|
||
return str;
|
||
}
|
||
|
||
function timestampForFileName() {
|
||
const d = new Date();
|
||
const pad = (n) => String(n).padStart(2, "0");
|
||
return `${d.getFullYear()}${pad(d.getMonth() + 1)}${pad(d.getDate())}-${pad(d.getHours())}${pad(d.getMinutes())}${pad(d.getSeconds())}`;
|
||
}
|
||
|
||
async function listCommand() {
|
||
const accessToken = await getAccessToken();
|
||
console.log("アクセス可能な掲示板一覧を取得します...");
|
||
const boards = await fetchAllBoards(accessToken);
|
||
console.log(`取得件数: ${boards.length} 件`);
|
||
|
||
const outputPath = path.join(__dirname, `board-list_${timestampForFileName()}.csv`);
|
||
const lines = ["boardId,boardName"];
|
||
for (const b of boards) {
|
||
lines.push(`${csvEscape(b.boardId)},${csvEscape(b.boardName)}`);
|
||
}
|
||
// ExcelでUTF-8を正しく開けるようBOM付きで出力
|
||
fs.writeFileSync(outputPath, "\uFEFF" + lines.join("\n"), "utf8");
|
||
|
||
console.log(`保存先: ${outputPath}`);
|
||
boards.forEach((b) => console.log(` - [${b.boardId}] ${b.boardName}`));
|
||
}
|
||
|
||
async function main() {
|
||
const [, , command, ...rest] = process.argv;
|
||
|
||
if (command === "sync") {
|
||
await syncCommand();
|
||
} else if (command === "list") {
|
||
await listCommand();
|
||
} else if (command === "test-recent") {
|
||
await testRecentCommand();
|
||
} else if (command === "ask") {
|
||
const question = rest.join(" ");
|
||
if (!question) {
|
||
console.error('使い方: node lineworks-anythingllm.js ask "問い合わせ内容"');
|
||
process.exitCode = 1;
|
||
return;
|
||
}
|
||
await askCommand(question);
|
||
} else {
|
||
console.log('使い方: node lineworks-anythingllm.js [sync|list|ask "質問文"]');
|
||
}
|
||
}
|
||
|
||
main().catch((err) => {
|
||
console.error("処理に失敗しました。");
|
||
console.error(err.message);
|
||
process.exitCode = 1;
|
||
}); |