Merge remote-tracking branch 'origin/master' into sso-exclusive-all

Author: haraldschilly

src/compute/compute/lib/listings.ts

@@ -52,8 +52,8 @@ export async function initListings({
     project_id,
     compute_server_id,
     getListing,
-    createWatcher: (path: string, debounceMs: number) =>
-      new Watcher(path, debounceMs),
+    createWatcher: (path: string, debounce: number) =>
+      new Watcher(path, { debounce }),
     onDeletePath: (path) => {
       logger.debug("onDeletePath -- TODO:", { path });
     },

src/compute/compute/package.json

@@ -42,7 +42,7 @@
     "cookie": "^0.5.0",
     "debug": "^4.3.2",
     "websocketfs": "^0.17.2",
-    "ws": "^8.13.0"
+    "ws": "^8.18.0"
   },
   "homepage": "https://github.com/sagemathinc/cocalc/tree/master/src/packages/compute",
   "repository": {

src/compute/pnpm-lock.yaml

@@ -13,7 +13,7 @@
     "psql": "export PGHOST=${PGHOST:=$INIT_CWD/data/postgres/socket}; PGUSER='smc' psql",
     "database": "cd dev/project && ./start_postgres.py",
     "c": "LOGS=/tmp/ DEBUG='cocalc:*' ./scripts/c",
-    "version-check": "pip3 install typing_extensions mypy && ./workspaces.py version-check && mypy scripts/check_npm_packages.py",
+    "version-check": "pip3 install typing_extensions mypy  || pip3 install --break-system-packages typing_extensions mypy && ./workspaces.py version-check && mypy scripts/check_npm_packages.py",
     "test-parallel": "unset DEBUG && pnpm run version-check && cd packages && pnpm run -r --parallel test",
     "test": "unset DEBUG && pnpm run version-check && cd packages && pnpm run -r test",
     "prettier-all": "cd packages/"

src/package.json

@@ -10,7 +10,7 @@ the settings of your project.
 ```sh
 export PROJECT_ID=6640ddad-4bdd-4745-8e63-8db74686a20e
 export API_KEY=sk-FcnRs3NxsTZROgbF000001
-export API_SERVER=http://localhost:9001
+export API_SERVER=http://127.0.0.1:9001
 ```
 
 The run code in node from the shell in the current directory:

src/packages/api-client/README.md

@@ -4,6 +4,8 @@
  */
 
 process.env.COCALC_PROJECT_MONITOR_INTERVAL_S = "1";
+// default is much lower, might fail if you have more procs than the default
+process.env.COCALC_PROJECT_INFO_PROC_LIMIT = "10000";
 
 import { executeCode } from "./execute-code";
 
@@ -300,6 +302,147 @@ describe("async", () => {
   );
 });
 
+// the await case is essentially like the async case above, but it will block for a bit
+describe("await", () => {
+  const check = (s) => {
+    expect(s.type).toEqual("async");
+    if (s.type !== "async") return;
+    expect(s.status).toEqual("completed");
+    expect(s.elapsed_s).toBeGreaterThan(1);
+    expect(s.elapsed_s).toBeLessThan(3);
+    expect(s.exit_code).toBe(0);
+    expect(s.pid).toBeGreaterThan(1);
+    expect(s.stdout).toEqual("foo\n");
+    expect(s.stderr).toEqual("");
+  };
+
+  it("returns when a job finishes", async () => {
+    const c = await executeCode({
+      command: "sleep 2; echo 'foo'",
+      bash: true,
+      err_on_exit: false,
+      async_call: true,
+    });
+    expect(c.type).toEqual("async");
+    if (c.type !== "async") return;
+    const { status, job_id, pid } = c;
+    expect(status).toEqual("running");
+    expect(pid).toBeGreaterThan(1);
+    const t0 = Date.now();
+    const s = await executeCode({
+      async_await: true,
+      async_get: job_id,
+      async_stats: true,
+    });
+    const t1 = Date.now();
+    // This is the main test: it really waited for at least a second until the job completed
+    expect((t1 - t0) / 1000).toBeGreaterThan(1);
+    check(s);
+    if (s.type !== "async") return;
+    expect(Array.isArray(s.stats)).toBeTruthy();
+  });
+
+  it("returns immediately if already done", async () => {
+    const c = await executeCode({
+      command: "sleep 1.1; echo 'foo'",
+      bash: true,
+      err_on_exit: false,
+      async_call: true,
+    });
+    expect(c.type).toEqual("async");
+    if (c.type !== "async") return;
+    const { status, job_id, pid } = c;
+    expect(status).toEqual("running");
+    expect(pid).toBeGreaterThan(1);
+    await new Promise((done) => setTimeout(done, 2000));
+    const s = await executeCode({
+      async_await: true,
+      async_get: job_id,
+      async_stats: true,
+    });
+    check(s);
+    if (s.type !== "async") return;
+    expect(s.elapsed_s).toBeLessThan(1.5);
+  });
+
+  it("deal with unknown executables", async () => {
+    const c = await executeCode({
+      command: "random123unknown99",
+      err_on_exit: false,
+      async_call: true,
+    });
+    expect(c.type).toEqual("async");
+    if (c.type !== "async") return;
+    const { job_id, pid } = c;
+    expect(pid).toBeUndefined();
+    const s = await executeCode({
+      async_await: true,
+      async_get: job_id,
+      async_stats: true,
+    });
+    expect(s.type).toEqual("async");
+    if (s.type !== "async") return;
+    expect(s.exit_code).toBe(1);
+    expect(s.stderr).toContain("ENOENT");
+    expect(s.status).toBe("error");
+  });
+
+  it("returns an error", async () => {
+    const c = await executeCode({
+      command: "sleep .1; >&2 echo baz; exit 3",
+      bash: true,
+      err_on_exit: false,
+      async_call: true,
+    });
+    expect(c.type).toEqual("async");
+    if (c.type !== "async") return;
+    const { status, job_id, pid } = c;
+    expect(status).toEqual("running");
+    expect(pid).toBeGreaterThan(1);
+    const t0 = Date.now();
+    const s = await executeCode({
+      async_await: true,
+      async_get: job_id,
+      async_stats: true,
+    });
+    expect((Date.now() - t0) / 1000).toBeGreaterThan(0.05);
+    expect(s.type).toEqual("async");
+    if (s.type !== "async") return;
+    expect(s.stderr).toEqual("baz\n");
+    expect(s.exit_code).toEqual(3);
+    expect(s.status).toEqual("completed");
+  });
+
+  it("react to a killed process", async () => {
+    const c = await executeCode({
+      command: "sh",
+      args: ["-c", `echo foo; sleep 1; echo bar;`],
+      bash: false,
+      err_on_exit: false,
+      async_call: true,
+    });
+    expect(c.type).toEqual("async");
+    if (c.type !== "async") return;
+    const { job_id, pid } = c;
+    await new Promise((done) => setTimeout(done, 100));
+    await executeCode({
+      command: `kill -9 -${pid}`,
+      bash: true,
+    });
+    const s = await executeCode({
+      async_await: true,
+      async_get: job_id,
+      async_stats: true,
+    });
+    expect(s.type).toEqual("async");
+    if (s.type !== "async") return;
+    expect(s.stderr).toEqual("");
+    expect(s.stdout).toEqual("foo\n");
+    expect(s.exit_code).toEqual(0);
+    expect(s.status).toEqual("completed");
+  });
+});
+
 // we burn a bit of CPU to get the cpu_pct and cpu_secs up
 const CPU_PY = `
 from time import time