Merge pull request #10 from benthecoder/clustering-viz

Clustering viz

Author: benthecoder

app/api/embeddings/route.ts

@@ -0,0 +1,136 @@
+import { sql } from "@vercel/postgres";
+import { NextResponse } from "next/server";
+import { computeUMAP, normalizePositions } from "@/utils/umapUtils";
+
+interface ChunkRow {
+  id: string;
+  post_slug: string;
+  post_title: string;
+  content: string;
+  chunk_type: string;
+  metadata: {
+    published_date?: string;
+    tags?: string[];
+  };
+  sequence: number;
+  embedding: unknown;
+  created_at: string;
+}
+
+interface ArticleData {
+  id: string;
+  postSlug: string;
+  postTitle: string;
+  content: string;
+  chunkType: string;
+  metadata: ChunkRow["metadata"];
+  sequence: number;
+  embedding: number[];
+  publishedDate?: string;
+  tags: string[];
+  createdAt: string;
+  index: number;
+  x: number;
+  y: number;
+}
+
+const parseEmbedding = (embedding: unknown): number[] => {
+  if (Array.isArray(embedding)) {
+    return embedding;
+  }
+
+  if (typeof embedding === "string") {
+    try {
+      const parsed = JSON.parse(embedding);
+      if (Array.isArray(parsed)) {
+        return parsed;
+      }
+    } catch {
+      // Parse PostgreSQL vector format
+      const cleaned = embedding.replace(/[\[\]]/g, "");
+      return cleaned.split(",").map(Number);
+    }
+  }
+
+  return [];
+};
+
+export async function GET(request: Request) {
+  try {
+    const { searchParams } = new URL(request.url);
+    const nNeighbors = parseInt(searchParams.get("neighbors") || "8");
+    const minDist = parseFloat(searchParams.get("minDist") || "0.05");
+    const spread = parseFloat(searchParams.get("spread") || "2.0");
+
+    const results = await sql<ChunkRow>`
+      SELECT DISTINCT ON (post_slug)
+        id,
+        post_slug,
+        post_title,
+        content,
+        chunk_type,
+        metadata,
+        sequence,
+        embedding,
+        created_at
+      FROM content_chunks
+      WHERE embedding IS NOT NULL
+      ORDER BY
+        post_slug,
+        CASE WHEN chunk_type = 'full-post' THEN 0 ELSE 1 END,
+        sequence
+    `;
+
+    const parsedData = results.rows
+      .map((row, index) => ({
+        id: row.id,
+        postSlug: row.post_slug,
+        postTitle: row.post_title,
+        content: row.content,
+        chunkType: row.chunk_type,
+        metadata: row.metadata,
+        sequence: row.sequence,
+        embedding: parseEmbedding(row.embedding),
+        publishedDate: row.metadata?.published_date,
+        tags: row.metadata?.tags || [],
+        createdAt: row.created_at,
+        index,
+      }))
+      .filter((item) => item.embedding.length > 0);
+
+    const embeddings = parsedData.map((item) => item.embedding);
+    const umapPositions = computeUMAP(embeddings, {
+      nNeighbors: Math.min(nNeighbors, parsedData.length - 1),
+      minDist,
+      spread,
+    });
+
+    const normalizedPositions = normalizePositions(
+      umapPositions,
+      1000,
+      1000,
+      50
+    );
+
+    const processedData: ArticleData[] = parsedData.map((item, index) => ({
+      ...item,
+      x: normalizedPositions[index].x,
+      y: normalizedPositions[index].y,
+    }));
+
+    return NextResponse.json({
+      success: true,
+      data: processedData,
+      count: processedData.length,
+    });
+  } catch (error) {
+    console.error("Error fetching embeddings:", error);
+    return NextResponse.json(
+      {
+        error: "Failed to fetch embeddings data",
+        details: error instanceof Error ? error.message : "Unknown error",
+      },
+      { status: 500 }
+    );
+  }
+}

app/posts/page.tsx

@@ -49,25 +49,21 @@ const ArchivePage = async ({
             </Link>
           </div>
           <div className="text-black-400">•</div>
-          <div>
-            <Link
-              href="/random"
-              className="underline hover:text-light-accent dark:hover:text-dark-accent transition-colors"
-            >
-              random
-            </Link>{" "}
-            🎲
-          </div>
-          <div className="text-black-400">•</div>
           <div>
             <Link
               href="/search"
               className="underline hover:text-light-accent dark:hover:text-dark-accent transition-colors"
             >
               search
             </Link>{" "}
-            🔍
           </div>
+          <div className="text-black-400">•</div>
+          <Link
+            href="/viz"
+            className="underline hover:text-light-accent dark:hover:text-dark-accent transition-colors"
+          >
+            viz
+          </Link>
         </div>
 
         <p className="text-japanese-sumiiro dark:text-japanese-murasakisuishiyou text-sm mb-4 font-medium">

app/search/page.tsx

@@ -305,7 +305,7 @@ function SearchContent() {
         hasSearched &&
         query && (
           <div className="text-center py-6 text-light-text/70 dark:text-dark-text/70">
-            <p>No results found for "{query}"</p>
+            <p>No results found for &quot;{query}&quot;</p>
             <p className="text-sm mt-2">
               Try a different search type or modify your query
             </p>

app/viz/page.tsx

@@ -0,0 +1,20 @@
+import KnowledgeMap from "../../components/KnowledgeMap";
+
+export const metadata = {
+  title: "knowledge map",
+  description: "semantic relationships across my writing",
+};
+
+export default function VisualizationPage() {
+  return (
+    <div>
+      <h1 className="font-bold text-left mb-6 text-2xl hover:text-light-accent dark:hover:text-dark-accent transition-colors">
+        knowledge map
+      </h1>
+
+      <div className="h-[75vh] rounded-lg overflow-hidden border border-gray-200 dark:border-gray-700 shadow-sm">
+        <KnowledgeMap className="w-full h-full" />
+      </div>
+    </div>
+  );
+}