Moved node schema ingestion service into a microservice

README.md

@@ -148,12 +148,34 @@
 <image src="./readme/cards/demo.svg"/>
 <br/>
 <br/>
-<div style="display: flex; gap: 16px;">
-  <img src="./readme/demo/demo-copilot-1.gif" alt="Copilot Demo 1"
-       style="width: 50%; border-radius: 8px; box-shadow: 0 2px 8px rgba(0,0,0,0.1);" />
-  <img src="./readme/demo/demo-copilot-2.gif" alt="Copilot Demo 2"
-       style="width: 50%; border-radius: 8px; box-shadow: 0 2px 8px rgba(0,0,0,0.1);" />
-</div>
+<h3>Simple workflow generation</h3>
+
+<table>
+  <tr>
+    <td align="center" width="50%">
+      <img src="./readme/demo/demo-copilot-1.gif" width="100%" style="border-radius:8px;" />
+    </td>
+    <td align="center" width="50%">
+      <img src="./readme/demo/demo-copilot-2.gif" width="100%" style="border-radius:8px;" />
+    </td>
+  </tr>
+</table>
+
+<br/>
+
+<h3>Complicated workflow generation</h3>
+
+<table>
+  <tr>
+    <td align="center" width="50%">
+      <img src="./readme/demo/demo-copilot-3.gif" width="100%" style="border-radius:8px;" />
+    </td>
+    <td align="center" width="50%">
+      <img src="./readme/demo/demo-copilot-4.gif" width="100%" style="border-radius:8px;" />
+    </td>
+  </tr>
+</table>
+
 <br/>
 <br/>
 <br/>
@@ -275,6 +297,34 @@ git merge main
 git push
 ```
 
+## Graph Example:
+
+```
+main
+ ↑
+ │ (merge when dev is stable)
+ │
+dev
+ ├── chore/project-scaffold
+ ├── chores/DeploymentPreperation
+ ├── feat/Microservices
+ │   └── feat/Microservices-SchemaExtractor-AST
+ │
+ ↑ (merge back to main)
+ │
+ ├── feature/Client
+ │   ├── feature/Client-copilot
+ │   └── feature/Client-landingPage
+ ├── feature/Copilot
+ │   └── feature/Copilot-langChain
+ ├── feature/Server-Client/Posting-workflows
+ │
+ ↑ (merge back to main)
+ │
+ ├── feature/Server-Client/userProfile
+ ├── refactor/Server-Client/cleaning
+ └── refactor/Server-copilot
+```
 ---
 
 <h3>Why This Workflow?</h3>
@@ -315,6 +365,20 @@ Every major feature was developed alongside tests to ensure reliability, prevent
   </div>
 </div>
 
+<br />
+
+<h3>Feature tests</h3>
+<div style="background-color: #000000; padding: 20px; border-radius: 8px; overflow-y: auto; margin: 20px 0; max-height: 600px;">
+  <div style="display: flex; flex-direction: column; gap: 20px;">
+    <img src="./readme/testing/testing-feat-1.png" alt="Testing 1" style="width: 100%; height: auto; border-radius: 6px;">
+    <img src="./readme/testing/testing-feat-2.png" alt="Testing 2" style="width: 100%; height: auto; border-radius: 6px;">
+    <img src="./readme/testing/testing-feat-3.png" alt="Testing 3" style="width: 100%; height: auto; border-radius: 6px;">
+    <img src="./readme/testing/testing-feat-4.png" alt="Testing 4" style="width: 100%; height: auto; border-radius: 6px;">
+  </div>
+</div>
+
+## 📄 License
 
-<h3>MIT LICENSE</h3>
 ![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)
+This project is licensed under the MIT License.
+

microservice/ast-Schema-Extractor/.gitignore

@@ -0,0 +1,2 @@
+node_modules
+n8n-nodes-base

microservice/ast-Schema-Extractor/ast/arrays.ts

@@ -0,0 +1,44 @@
+import { Node, ArrayLiteralExpression } from "ts-morph";
+import { resolveInitializer } from "./resolver";
+
+/**
+ * Resolves an array-like AST node into its concrete elements.
+ *
+ * Handles:
+ * - identifiers pointing to arrays
+ * - spread elements (...otherArray)
+ * - nested indirections
+ *
+ * Always returns a flat list of Nodes.
+ */
+export function resolveArrayElements(node?: Node | null): Node[] {
+  if (!node) return [];
+
+  const resolved = resolveInitializer(node) ?? node;
+  const result: Node[] = [];
+
+  if (Node.isArrayLiteralExpression(resolved)) {
+    for (const element of resolved.getElements()) {
+      if (Node.isSpreadElement(element)) {
+        const inner = resolveInitializer(element.getExpression());
+        if (inner) {
+          result.push(...resolveArrayElements(inner));
+        }
+      } else {
+        result.push(element);
+      }
+    }
+    return result;
+  }
+
+  // Identifier pointing to array defined elsewhere
+  if (Node.isIdentifier(resolved) || Node.isPropertyAccessExpression(resolved)) {
+    const symbol = resolved.getSymbol?.();
+    const declaration = symbol?.getDeclarations()?.[0] as any;
+    const init = declaration?.getInitializer?.();
+
+    if (init) return resolveArrayElements(init);
+  }
+
+  return result;
+}

microservice/ast-Schema-Extractor/ast/literals.ts

@@ -0,0 +1,32 @@
+import { Node, SyntaxKind } from "ts-morph";
+
+/**
+ * Extracts a primitive value from an AST node.
+ *
+ * Supported:
+ * - string
+ * - number
+ * - boolean
+ *
+ * Anything else falls back to text representation.
+ */
+export function extractLiteral(node?: Node | null): any {
+  if (!node) return undefined;
+
+  if (node.getKind() === SyntaxKind.StringLiteral) {
+    const text = node.getText();
+    return text.slice(1, -1); // remove quotes
+  }
+
+  if (node.getKind() === SyntaxKind.NumericLiteral) {
+    return Number(node.getText());
+  }
+
+  if (node.getKind() === SyntaxKind.TrueKeyword) return true;
+  if (node.getKind() === SyntaxKind.FalseKeyword) return false;
+
+  // Fallback: return sanitized text
+  return node.getText
+    ? node.getText().replace(/['"`]/g, "")
+    : undefined;
+}

microservice/ast-Schema-Extractor/ast/objects.ts

@@ -0,0 +1,21 @@
+import { Node, ObjectLiteralExpression } from "ts-morph";
+
+/**
+ * Safely retrieves a property initializer from an object literal.
+ *
+ * Only supports PropertyAssignment.
+ * Shorthand and spreads are ignored intentionally.(handled by arrays resolver)
+ */
+export function getObjectProperty(
+  obj: ObjectLiteralExpression,
+  name: string
+): Node | undefined {
+  const prop = obj.getProperty(name);
+  if (!prop) return undefined;
+
+  if (Node.isPropertyAssignment(prop)) {
+    return prop.getInitializer();
+  }
+
+  return undefined;
+}

microservice/ast-Schema-Extractor/ast/paths.ts

@@ -0,0 +1,14 @@
+import path from "path";
+
+/**
+ * Cross-platform safe path prefix comparison.
+ *
+ * Needed because ts-morph returns absolute paths
+ * and Windows path separators will break naive checks.
+ */
+export function pathStartsWith(candidate: string, prefix: string): boolean {
+  const a = path.resolve(candidate).split(path.sep);
+  const b = path.resolve(prefix).split(path.sep);
+
+  return a.slice(0, b.length).join(path.sep) === b.join(path.sep);
+}

microservice/ast-Schema-Extractor/ast/resolver.ts

@@ -0,0 +1,38 @@
+import { Node, SyntaxKind } from "ts-morph";
+
+/**
+ * Resolves a node to its actual initializer if it is:
+ * - an Identifier
+ * - a PropertyAccessExpression
+ *
+ * This follows imports and exported constants across files.
+ *
+ * If the node is already a literal/object/array, it is returned as-is.
+ *
+ */
+export function resolveInitializer(node?: Node | null): Node | undefined {
+  if (!node) return undefined;
+
+  // Identifiers or property accesses may point to imported constants
+  if (Node.isIdentifier(node) || Node.isPropertyAccessExpression(node)) {
+    const symbol = node.getSymbol?.();
+    if (!symbol) return undefined;
+
+    const declaration = symbol.getDeclarations()?.[0];
+    if (!declaration) return undefined;
+
+    // Variable declarations usually have initializers
+    if (
+      "getInitializer" in declaration &&
+      typeof (declaration as any).getInitializer === "function"
+    ) {
+      return (declaration as any).getInitializer() ?? declaration;
+    }
+
+    // Fallback: return declaration itself
+    return declaration;
+  }
+
+  // Already resolved
+  return node;
+}

microservice/ast-Schema-Extractor/config/paths.ts

@@ -0,0 +1,13 @@
+import path from "path";
+
+/**
+ * Centralized filesystem paths.
+ * Keeps environment-specific concerns out of logic.
+ */
+export const ROOT = path.resolve("./n8n-nodes-base/packages/nodes-base");
+
+export const NODES_DIR = path.join(ROOT, "nodes");
+export const CREDS_DIR = path.join(ROOT, "credentials");
+
+export const OUTPUT_FILE = "n8n_node_schemas.json";
+export const LOG_FILE = "harvest.log";

microservice/ast-Schema-Extractor/domain/cache/connectionTypes.ts

@@ -0,0 +1,65 @@
+import { Project } from "ts-morph";
+
+/**
+ * Cache for NodeConnectionTypes → friendly name mapping.
+ *
+ * This is loaded once per process and reused.
+ *
+ * Example:
+ *   NodeConnectionTypes.Main → "Main"
+ */
+export class ConnectionTypeCache {
+  private readonly map: Record<string, string> = {};
+  private loaded = false;
+
+  constructor(private readonly project: Project) {}
+
+  /**
+   * Loads NodeConnectionTypes from the project AST.
+   * Safe to call multiple times.
+   */
+  load(): void {
+    if (this.loaded) return;
+
+    for (const sourceFile of this.project.getSourceFiles()) {
+      if (!sourceFile.getFullText().includes("NodeConnectionTypes")) continue;
+
+      // Prefer enum definition
+      const enums = sourceFile
+        .getEnums()
+        .filter(e => e.getName() === "NodeConnectionTypes");
+
+      if (enums.length) {
+        for (const member of enums[0].getMembers()) {
+          const name = member.getName();
+          this.map[`NodeConnectionTypes.${name}`] = name;
+        }
+        this.loaded = true;
+        return;
+      }
+
+      // Fallback: const object literal
+      const text = sourceFile.getFullText();
+      const match = text.match(/NodeConnectionTypes\s*=\s*\{([\s\S]*?)\}/m);
+      if (match) {
+        for (const line of match[1].split("\n")) {
+          const m = line.match(/(\w+)\s*:/);
+          if (m) {
+            this.map[`NodeConnectionTypes.${m[1]}`] = m[1];
+          }
+        }
+        this.loaded = true;
+        return;
+      }
+    }
+
+    this.loaded = true;
+  }
+
+  /**
+   * Resolves a raw connection type value into a friendly name.
+   */
+  resolve(raw: string): string {
+    return this.map[raw] ?? raw.replace(/^NodeConnectionTypes\./, "");
+  }
+}