diff --git a/.github/workflows/make-and-test.yml b/.github/workflows/make-and-test.yml index 88d31eaff7..96da238ff2 100644 --- a/.github/workflows/make-and-test.yml +++ b/.github/workflows/make-and-test.yml @@ -65,7 +65,6 @@ jobs: matrix: # See supported Node.js release schedule at https://nodejs.org/en/about/releases/ node-version: - - "18" - "20" pg-version: - "13.12" @@ -92,7 +91,22 @@ jobs: # cache: "pnpm" # cache-dependency-path: "src/packages/pnpm-lock.yaml" + - name: Download and install Valkey + run: | + VALKEY_VERSION=8.1.2 + curl -LO https://download.valkey.io/releases/valkey-${VALKEY_VERSION}-jammy-x86_64.tar.gz + tar -xzf valkey-${VALKEY_VERSION}-jammy-x86_64.tar.gz + sudo cp valkey-${VALKEY_VERSION}-jammy-x86_64/bin/valkey-server /usr/local/bin/ + + - name: Set up Python venv and Jupyter kernel + run: | + python3 -m pip install --upgrade pip virtualenv + python3 -m virtualenv venv + source venv/bin/activate + pip install ipykernel + python -m ipykernel install --prefix=./jupyter-local --name python3-local --display-name "Python 3 (Local)" + - run: cd src && npm install -g pnpm - run: cd src && pnpm run make - - run: cd src && pnpm nats-server-daemon - - run: cd src && pnpm run test + - run: source venv/bin/activate && cd src && pnpm run test || pnpm run test || pnpm run test + - run: cd src && pnpm run depcheck diff --git a/.gitignore b/.gitignore index 3683eb4a65..7f06ad5a5c 100644 --- a/.gitignore +++ b/.gitignore @@ -157,7 +157,5 @@ src/packages/frontend/i18n/extracted.json src/packages/frontend/i18n/trans/*.compiled.json **/*.db - -src/packages/project/nats/project-env.sh - +**/project-env.sh **/*.bash_history diff --git a/docs/nats/devlog.md b/docs/nats/devlog.md deleted file mode 100644 index f1d89fe55e..0000000000 --- a/docs/nats/devlog.md +++ /dev/null @@ -1,299 +0,0 @@ -# NATS Development and Integration Log - -## [x] Goal: nats from nodejs - -- start a nats server in cocalc\-docker -- connect from nats cli outside docker -- connect to it from the nodejs client over a websocket - -```sh -nats-server -p 5004 - -nats context save --select --server nats://localhost:5004 nats - -nats sub '>' -``` - -Millions of messages a second works \-\- and you can run like 5x of these at once without saturating nats\-server. - -```js -import { connect, StringCodec } from "nats"; -const nc = await connect({ port: 5004 }); -console.log(`connected to ${nc.getServer()}`); -const sc = StringCodec(); - -const t0 = Date.now(); -for (let i = 0; i < 1000000; i++) { - nc.publish("hello", sc.encode("world")); -} -await nc.drain(); -console.log(Date.now() - t0); -``` - -That was connecting over TCP. Now can we connect via websocket? - -## [x] Goal: Websocket from browser - -First need to start a nats **websocket** server instead on port 5004: - -[https://nats.io/blog/getting\-started\-nats\-ws/](https://nats.io/blog/getting-started-nats-ws/) - -```sh -nats context save --select --server ws://localhost:5004 ws -~/nats/nats.js/lib$ nats context select ws -NATS Configuration Context "ws" - - Server URLs: ws://localhost:5004 - Path: /projects/3fa218e5-7196-4020-8b30-e2127847cc4f/.config/nats/context/ws.json - -~/nats/nats.js/lib$ nats pub foo bar -21:24:53 Published 3 bytes to "foo" -~/nats/nats.js/lib$ -``` - -## - -- their no\-framework html example DOES work for me! -- [https://localhost:4043/projects/3fa218e5\-7196\-4020\-8b30\-e2127847cc4f/files/nats/nats.js/lib/ws.html](https://localhost:4043/projects/3fa218e5-7196-4020-8b30-e2127847cc4f/files/nats/nats.js/lib/ws.html) -- It takes about 1\-2 seconds to send **one million messages** from browser outside docker to what is running inside there! - -## [x] Goal: actually do something useful - -- nats server -- browser connects via websocket port 5004 -- nodejs hub connects via tcp -- hub answers a ping or something else from the browser... - -This worked perfectly with no difficulty. It's very fast and flexible and robust. - -Reconnects work, etc. - -## [x] Goal: proxying - -- nats server with websocket listening on localhost:5004 -- proxy it via node\-proxy in the hub to localhost:4043/nats -- as above - -This totally worked! - -Everything is working that I try?! - -Maybe NATS totally kicks ass. - -## [x] Goal: do something actually useful. - -- authentication: is there a way to too who the user who made the websocket connection is? - - worry about this **later** \- obviously possible and not needed for a POC -- let's try to make `write_text_file_to_project` also be possible via nats. -- OK, made some of api/v2 usable. Obviously this is really minimal POC. - -## [x] GOAL: do something involving the project - -The most interesting use case for nats/jetsteam is timetravel collab editing, where this is all a VERY natural fit. - -But for now, let's just do _something_ at all. - -This worked - I did project exec with subject projects.{project_id}.api - -## [x] Goal: Queue group for hub api - -- change this to be a queue group and test by starting a few servers at once - -## [x] Goal: Auth Strategy that is meaningful - -Creating a creds file that encodes a JWT that says what you can publish and subscribe to, then authenticating with that works. - -- make it so user with account_id can publish to hub.api.{account_id} makes it so we know the account_id automatically by virtue of what was published to. This works. - -## [x] Goal: Solve Critical Auth Problems - -Now need to solve two problems: - -- [x] GOAL: set the creds for a browser client in a secure http cookie, so the browser can't directly access it - -I finally figured this out after WASTING a lot of time with stupid AI misleading me and trying actively to get me to write very stupid insecure code as a lazy workaround. AI really is very, very dangerous... The trick was to read the docs repeatedly, increase logging a lot, and \-\- most imporantly \-\- read the relevant Go source code of NATS itself. The answer is to modify the JWT so that it explicitly has bearer set: `nsc edit user wstein --bearer` - -This makes it so the server doesn't check the signature of the JWT against the _user_ . Putting exactly the JWT token string in the cookie then works because "bearer" literally tells the backend server not to do the signature check. I think this is secure and the right approach because the server checks that the JWT is valid using the account and operator signatures. - -**WAIT!** Using signing keys [https://docs.nats.io/using\-nats/nats\-tools/nsc/signing_keys](https://docs.nats.io/using-nats/nats-tools/nsc/signing_keys) \(and https://youtu.be/KmGtnFxHnVA?si=0uvLMBTJ5TUpem4O \) is VASTLY superior. There's just one JWT issued to each user, and we make a server\-side\-only JWT for their account that has everything. The user never has to reconnect or change their JWT. We can adjust the subject on the fly to account for running projects \(or collaboration changes\) at any time server side. Also the size limits go away, so we don't have to compress project_id's \(probably\). - -## Goal: Implement Auth Solution for Browsers - -- [x] automate creation of creds for browser clients, i.e., what we just did with the nsc tool manually -- - ---- - -This is my top priority goal for NOW! - -What's the plan? - -Need to figure out how to do all the nsc stuff from javascript, storing results in the database? - -- Question: how do we manage creating signing keys and users from nodejs? Answer: clear from many sources that we must use the nsc CLI tool via subprocess calls. Seems fine to me. -- [x] When a user signs in, we check for their JWT in the database. If it is there, set the cookie. If not, create the signing key and JWT for them, save in database, and set the cookie. -- [x] update nats\-server resolver state after modifying signing cookie's subjects configuration. - -``` -nsc edit operator --account-jwt-server-url nats://localhost:4222 -``` - -Now I can do `nsc push` and it just works. - -[x] TODO: when signing out, need to delete the jwt cookie or dangerous private info leaks... and also new info not set properly. - -- [x] similar creds for projects, I.e., access to a project means you can publish to `projects.{project_id}.>` Also, projects should have access to something under hub. - -## [x] Goal: Auth for Projects - -Using an env variable I got a basic useful thing up and running. - ---- - -Some thoughts about project auth security: - -- [ ] when collaborators on a project leave maybe we change JWT? Otherwise, in theory any user of a project can probably somehow get access to the project's JWT \(it's in memory at least\) and still act as the project. Changing JWT requires reconnect. This could be "for later", since even now we don't have this level of security! -- [ ] restarting project could change JWT. That's like the current project's secret token being changed. - -## [ ] Goal: nats-server automation of creation and configuration of system account, operator, etc. - -- This looks helpful: https://www.synadia.com/newsletter/nats-weekly-27/ -- NOT DONE YET - -## [x] Goal: Terminal! Something complicated involving the project which is NOT just request/response - -- Implementing terminals goes beyond request/response. -- It could also leverage jetstream if we want for state (?). -- Multiple connected client - -Project/compute server sends terminal output to - - project.{project_id}.terminal.{sha1(path)} - -Anyone who can read project gets to see this. - -Browser sends terminal input to - - project.{project_id}.{group}.{account_id}.terminal.{sha1(path)} - -API calls: - - - to start terminal - - to get history (move to jetstream?) - -If I can get this to work, then collaborative editing and everything else is basically the same (just more details). - -## [x] Goal: Terminal! #now - -Make it so an actual terminal works, i.e., UI integration. - -## [x] Goal: Terminal JetStream state - -Use Jetstream to store messages from terminal, so user can reconnect without loss. !? This is very interesting... - -First problem -- we used the system account SYS for all our users; however, -SYS can't use jetstreams, as explained here https://github.com/nats-io/nats-server/discussions/6033 - -Let's redo *everything* with a new account called "cocalc". - -```sh -~/nats$ nsc create account --name=cocalc -[ OK ] generated and stored account key "AD4G6R62BDDQUSCJVLZNA7ES7R3A6DWXLYUWGZV74EJ2S6VBC7DQVM3I" -[ OK ] added account "cocalc" -~/nats$ nats context save admin --creds=/projects/3fa218e5-7196-4020-8b30-e2127847cc4f/.local/share/nats/nsc/keys/creds/MyOperator/cocalc/admin.creds -~/nats$ nsc edit account cocalc --js-enable 1 -~/nats$ nsc push -a cocalc -``` - -```js -// making the stream for ALL terminal activity -await jsm.streams.add({ name: 'project-81e0c408-ac65-4114-bad5-5f4b6539bd0e-terminal', subjects: ['project.81e0c408-ac65-4114-bad5-5f4b6539bd0e.terminal.>'] }); - -// making a consumer for just one subject (e.g., one terminal frame) -z = await jsm.consumers.add('project-81e0c408-ac65-4114-bad5-5f4b6539bd0e-terminal',{name:'9149af7632942a94ea13877188153bd8bf2ace57',filter:['project.81e0c408-ac65-4114-bad5-5f4b6539bd0e.terminal.9149af7632942a94ea13877188153bd8bf2ace57']}) -c = await js.consumers.get('project-81e0c408-ac65-4114-bad5-5f4b6539bd0e-terminal', '9149af7632942a94ea13877188153bd8bf2ace57') -for await (const m of await c.consume()) { console.log(cc.client.nats_client.jc.decode(m.data))} -``` - -NOTE!!! The above consumer is ephemeral -- it disappears if we don't grab it via c within a few seconds!!!! https://docs.nats.io/using-nats/developer/develop_jetstream/consumers - -## [ ] Goal: Jetstream permissions - -- [x] project should set up the stream for capturing terminal outputs. -- [x] delete old messages with a given subject. `nats stream purge project-81e0c408-ac65-4114-bad5-5f4b6539bd0e-terminal --seq=7000` - - there is a setting max\_msgs\_per\_subject on a stream, so **we just set that and are done!** Gees. It is too easy. -- [x] handle the other messages like resize -- [x] need to move those other messages to a different subject that isn't part of the stream!! -- [ ] permissions for jetstream usage and access -- [ ] use non\-json for the data.... -- [ ] refactor code so basic parameters \(e.g., subject names, etc.\) are defined in one place that can be imported in both the frontend and backend. -- [ ] font size keyboard shortcut -- [ ] need a better algorithm for sizing since we don't know when a user disconnects! - - when one user proposes a size, all other clients get asked their current size and only those that respond matter. how to do this? - -## [ ] Goal: Basic Collab Document Editing - -Plan. - -- [x] Use a kv store hosted on nats to trac syncstring objects as before. This means anybody can participate \(browser, compute server, project\) without any need to contact the database, hence eliminating all proxying! - -[x] Next Goal \- collaborative file editing \-\- some sort of "proof of concept"! This requires implementing the "ordered patches list" but on jetstream. Similar to the nats SyncTable I wrote yesterday, except will use jetstream directly, since it is an event stream, after all. - -- [x] synctable\-stream: change to one big stream for the whole project but **consume** a specific subject in that stream? - -[ ] cursors \- an ephemeral table - ---- - -- [ ] Subject For Particular File: `project.${project_id}.patches.${sha1(path)}` -- [ ] Stream: Records everything with this subject `project.${project_id}.patches` -- [ ] It would be very nice if we can use the server assigned timestamps.... but probably not - - [ ] For transitioning and de\-archiving, there must be a way to do this, since they have a backup/restore process - -## [ ] Goal: PostgreSQL Changefeed Synctable - -This is critical to solve. This sucks now. This is key to eliminating "hub\-websocket". This might be very easy. Here's the plan: - -- [x] make a request/response listener that listens on hub.account.{account\_id} and hub.db.project.{project\_id} for a db query. -- [x] if changes is false, just responds with the result of the query. -- [ ] if changes is true, get kv store k named `account-{account_id}` or `project-{project_id}` \(which can be used by project or compute server\). - - let id be the sha1 hash of the query \(and options\) - - k.id.update is less than X seconds ago, do nothing... it's already being updated by another server. - - do the query to the database \(with changes true\) - - write the results into k under k.id.data.key = value. - - keep watching for changes so long as k.id.interest is at most n\*X seconds ago. - - Also set k.id.update to now. - - return id -- [ ] another message to `hub.db.{account_id}` which contains a list of id's. - - When get this one, update k.id.interest to now for each of the id's. - -With the above algorithm, it should be very easy to reimplement the client side of SyncTable. Moreover, there are many advantages: - -- For a fixed account\_id or project\-id, there's no extra work at all for 1 versus 100 of them. I.e., this is great for opening a bunch of distinct browser windows. -- If you refresh your browser, everything stays stable \-\- nothing changes at all and you instantly have your data. Same if the network drops and resumes. -- When implementing our new synctable, we can immediately start with the possibly stale data from the last time it was active, then update it to the correct data. Thus even if everything but NATS is done/unavailable, the experience would be much better. It's like "local first", but somehow "network mesh first". With a leaf node it would literally be local first. - ---- - -This is working well! - -TODO: - -- [x] build full proof of concept SyncTable on top of my current implementation of synctablekvatomic, to _make sure it is sufficient_ - - this worked and wasn't too difficult - -THEN do the following to make it robust and scalable - -- [ ] store in nats which servers are actively managing which synctables -- [ ] store in nats the client interest data, instead of storing it in memory in a server? i.e., instead of client making an api call, they could instead just update a kv and say "i am interested in this changefeed". This approach would make everything just keep working easily even as servers scale up/down/restart. - ---- - -## [ ] Goal: Terminal and **compute server** - -Another thing to do for compute servers: - -- use jetstream and KV to agree on _who_ is running the terminal? - -This is critical to see how easily we can support compute servers using nats + jetstream. - diff --git a/src/README.md b/src/README.md index 1b995976c7..77a1a147d4 100644 --- a/src/README.md +++ b/src/README.md @@ -2,7 +2,7 @@ **Updated: Feb 2025** -CoCalc is a pretty large and complicated project, and it will only work with the current standard LTS release of node.js \( at least 18.17.1\) and a recent version of [pnpm](https://pnpm.io/). Also, you will need a LOT of RAM, a minimum of 16 GB. **It's very painful to do development with less than 32 GB of RAM.** +CoCalc is a pretty large and complicated project, and it will only work with the current standard LTS release of node.js \( at least 18.17.1\) and a recent version of [pnpm](https://pnpm.io/). Also, you will need a LOT of RAM, a minimum of 16 GB. **It's very painful to do development with less than 32 GB of RAM.** **Node.js and NPM Version Requirements:** @@ -27,13 +27,13 @@ _Note that `nvm install` is only required the first time you run this command or npm install -g pnpm ``` -Alternatively, if you do not wish to install `pnpm` globally, you can run `npm install` to install it as a dev +Alternatively, if you do not wish to install `pnpm` globally, you can run `npm install` to install it as a dev dependency. **Python virtual environment** -Some features of CoCalc (e.g., file creation) require local Python modules to be installed. To create a [Python virtual -environment](https://docs.python.org/3/library/venv.html) from which to run these modules, run (from the `src` +Some features of CoCalc (e.g., file creation) require local Python modules to be installed. To create a [Python virtual +environment](https://docs.python.org/3/library/venv.html) from which to run these modules, run (from the `src` directory): ```sh @@ -53,16 +53,16 @@ To install required dependencies, run ``` **You must have your virtual environment activated when running the CoCalc Hub (via `pnpm hub`)!** If, on the other -hand, you prefer that development packages be installed globally, you can jump directly to the above `pip install` +hand, you prefer that development packages be installed globally, you can jump directly to the above `pip install` command outside the context of a virtual environment. ## Build and Start Launch the install and build **for doing development.** -If you export the PORT environment variable, that determines what port everything listens on. This determines subtle things about configuration, so do this once and for all in a consistent way. +If you export the PORT environment variable, that determines what port everything listens on. This determines subtle things about configuration, so do this once and for all in a consistent way. -CoCalc also runs a NATS server listening on two ports on localhost, one for TCP and one for WebSocket connections. To avoid conflicts, you can customize their ports by setting the environment variables `COCALC_NATS_PORT` (default 4222), and `COCALC_NATS_WS_PORT` (default 8443). +CoCalc also runs a NATS server listening on two ports on localhost, one for TCP and one for WebSocket connections. To avoid conflicts, you can customize their ports by setting the environment variables `COCALC_NATS_PORT` (default 4222), and `COCALC_NATS_WS_PORT` (default 8443). **Note**: If you installed `pnpm` locally (instead of globally), simply run `npm run` in place of `pnpm` to execute these commands via [NPM run scripts](https://docs.npmjs.com/cli/v10/using-npm/scripts). @@ -71,11 +71,10 @@ these commands via [NPM run scripts](https://docs.npmjs.com/cli/v10/using-npm/sc ~/cocalc/src$ pnpm build-dev ``` -This will do `pnpm install` for all packages, and also build the typescript/coffeescript, and anything else into a dist directory for each module. Once `pnpm build-dev` finishes successfully, you can start using CoCalc by starting the database, nats server and the backend hub in three terminals. \(Note that 'pnpm nats\-server' will download, install and configure NATS automatically.\) You can start the database, nats\-server and hub in any order. +This will do `pnpm install` for all packages, and also build the typescript code, and anything else into a dist directory for each module. Once `pnpm build-dev` finishes successfully, you can start using CoCalc by starting the database and the backend hub in two terminals. You can start the database and hub in any order. ```sh ~/cocalc/src$ pnpm database # in one terminal -~/cocalc/src$ pnpm nats-server # in one terminal ~/cocalc/src$ pnpm hub # in another terminal ``` @@ -107,8 +106,8 @@ If necessary, you can delete all the `node_modules` and `dist` directories in al The code of CoCalc is in NPM packages in the `src/packages/` subdirectory. To do development you need to ensure each of the following two services are running, as explained above: -1. **PostgreSQL** database \-\- a postgres instance started via `pnpm database` -2. **Hub** \-\- a nodejs instance started via `pnpm hub` +1. **PostgreSQL** database \-\- a postgres instance started via `pnpm database` +2. **Hub** \-\- a nodejs instance started via `pnpm hub` Optionally, you may also need to type `pnpm tsc` in packages that you're editing to watch for changes, compile using Typescript and show an errors. @@ -143,11 +142,11 @@ You can also just type `pnpm psql` : ~/cocalc/src$ pnpm psql ``` -NOTE: As of Jan 2023, CoCalc should fully work with any version of PostgreSQL from version 14.x onward. However, obviously at some point we will stop supporting PostgreSQL v 14. +NOTE: As of Jan 2023, CoCalc should fully work with any version of PostgreSQL from version 14.x onward. However, obviously at some point we will stop supporting PostgreSQL v 14. ### 2. More about Starting the Hub -The Hub is CoCalc's backend node.js server. You can start it from its package directory as follows: +The Hub is CoCalc's backend node.js server. You can start it from its package directory as follows: ```sh ~/cocalc/src/packages/hub$ pnpm hub-project-dev @@ -155,7 +154,7 @@ The Hub is CoCalc's backend node.js server. You can start it from its package d That will ensure the latest version of the hub Typescript and Coffeescript gets compiled, and start a new hub running in the foreground logging what is happening to the console _**and also logging to files in**_ `cocalc/src/data/logs/hub` . Hit Control\+C to terminate this server. If you change any code in `packages/hub`, you have to stop the hub, then start it again as above in order for the changes to take effect. -The hub itself is running two copies of webpack along with two separate "Hot Module Replacement" servers, etc. One is for the `/static` endpoint \(see packages/static and packages/frontend\) and the other is for the nextjs server \(see packages/next\). +The hub itself is running two copies of webpack along with two separate "Hot Module Replacement" servers, etc. One is for the `/static` endpoint \(see packages/static and packages/frontend\) and the other is for the nextjs server \(see packages/next\). ### 3. Building only what has changed @@ -165,7 +164,7 @@ The command `pnpm build (or build-dev)`, when run from the src directory, caches ~/cocalc/src/$ pnpm build --exclude=static ``` -This is useful if you pull in a git branch or switch to a different git branch, and have no idea which packages have changed. That said, it's always much safer to just do the following instead of relying on this: +This is useful if you pull in a git branch or switch to a different git branch, and have no idea which packages have changed. That said, it's always much safer to just do the following instead of relying on this: ```sh ~/cocalc/src/$ pnpm clean && pnpm make-dev @@ -185,14 +184,6 @@ which installs exactly the right packages, and builds the code. See `packages/backend/data.ts` . In particular, you can set BASE_PATH, DATA, PGHOST, PGDATA, PROJECTS, SECRETS to override the defaults. Data is stored in `cocalc/src/data/` by default. -For NATS when doing development, it can be useful to set these in .bashrc so NATS uses ports of your choosing that are available. - -```sh -export COCALC_NATS_SERVER_NAME=localhost -export COCALC_NATS_PORT=5007 -export COCALC_NATS_WS_PORT=5008 -``` - #### File System Build Caching There are two types of file system build caching. These greatly improve the time to compile typescript or start webpack between runs. However, in rare cases bugs may lead to weird broken behavior. Here's where the caches are, so you know how to clear them to check if this is the source of trouble. _As of now, I'm_ _**not**_ _aware of any bugs in file system caching._ @@ -231,4 +222,3 @@ Regarding VS Code, the relevant settings can be found by searching for "autosave There's some `@cocalc/` packages at [NPMJS.com](http://NPMJS.com). However, _**we're no longer using**_ _**them in any way**_, and don't plan to publish anything new unless there is a compelling use case. - diff --git a/src/compute/compute/README.md b/src/compute/compute/README.md index ec23542568..7683c4c1d6 100644 --- a/src/compute/compute/README.md +++ b/src/compute/compute/README.md @@ -1,5 +1,7 @@ # @cocalc/compute +NOTE: a lot of this is out of date with the NATS, then Conat rewrites. In most cases now a compute server is exactly the same as a project. It just has a compute_server_id that is positive instead of 0. That's it. + ## Goal The minimal goal of this package is to connect from a nodejs process to a cocalc project, open a Jupyter notebook sync session, and provide the output. I.e., instead of the project itself running a kernel and providing output, the kernel will be provided by whatever client is running this `@cocalc/compute` package! @@ -107,18 +109,3 @@ await require("@cocalc/compute").jupyter({ }); 0; ``` - -### Terminal - -You should open the notebook Untitled.ipynb on [cocalc.com](http://cocalc.com). -Then set all the above env variables in another terminal and run the following code in node.js. **Running of that first \(if you split frame\) command line terminal will then switch to your local machine.** - -```js -await require("@cocalc/compute").terminal({ - project_id: process.env.PROJECT_ID, - path: "term.term", - cwd: "/tmp/project", -}); -0; -``` - diff --git a/src/compute/compute/bin/start.js b/src/compute/compute/bin/start.js index 6e329c5d0f..b1505819c2 100755 --- a/src/compute/compute/bin/start.js +++ b/src/compute/compute/bin/start.js @@ -9,7 +9,7 @@ process.env.BASE_PATH = process.env.BASE_PATH ?? "/"; process.env.API_SERVER = process.env.API_SERVER ?? "https://cocalc.com"; process.env.API_BASE_PATH = process.env.API_BASE_PATH ?? "/"; -const { mountProject, jupyter, terminal } = require("@cocalc/compute"); +const { mountProject, jupyter } = require("@cocalc/compute"); const PROJECT_HOME = "/home/user"; @@ -58,15 +58,6 @@ async function main() { path: PROJECT_HOME, }); - if (process.env.TERM_PATH) { - console.log("Connecting to", process.env.TERM_PATH); - term = await terminal({ - project_id: process.env.PROJECT_ID, - path: process.env.TERM_PATH, - cwd: PROJECT_HOME, - }); - } - if (process.env.IPYNB_PATH) { console.log("Connecting to", process.env.IPYNB_PATH); kernel = await jupyter({ @@ -92,15 +83,6 @@ async function main() { ); } - if (process.env.TERM_PATH) { - console.log( - `Your terminal ${process.env.TERM_PATH} should be running in this container.`, - ); - console.log( - ` ${process.env.API_SERVER}/projects/${process.env.PROJECT_ID}/files/${process.env.TERM_PATH}`, - ); - } - console.log(`Your home directory is mounted at ${PROJECT_HOME}`); console.log("\nPress Control+C to exit."); }; diff --git a/src/compute/compute/lib/filesystem.ts b/src/compute/compute/lib/filesystem.ts index 53b37c71bc..972bab36a2 100644 --- a/src/compute/compute/lib/filesystem.ts +++ b/src/compute/compute/lib/filesystem.ts @@ -21,8 +21,8 @@ import { import { apiCall } from "@cocalc/api-client"; import sendFiles from "./send-files"; import getFiles from "./get-files"; -// ensure that the nats client is initialized so that syncfs can connect to nats properly. -import "@cocalc/project/nats"; +// ensure that the conat client is initialized so that syncfs can connect properly. +import "@cocalc/project/conat"; const logger = getLogger("compute:filesystem"); diff --git a/src/compute/compute/lib/index.ts b/src/compute/compute/lib/index.ts index 3098eada44..46900d38b1 100644 --- a/src/compute/compute/lib/index.ts +++ b/src/compute/compute/lib/index.ts @@ -1,5 +1,4 @@ export { mountProject } from "./filesystem"; export { tasks } from "./tasks"; export { jupyter } from "./jupyter"; -export { terminal } from "./terminal"; export { manager } from "./manager"; diff --git a/src/compute/compute/lib/manager.ts b/src/compute/compute/lib/manager.ts index b34680032e..3a6ff4ad45 100644 --- a/src/compute/compute/lib/manager.ts +++ b/src/compute/compute/lib/manager.ts @@ -7,7 +7,7 @@ The manager does the following: */ import debug from "debug"; -import startProjectServers from "@cocalc/project/nats"; +import startProjectServers from "@cocalc/project/conat"; import { pingProjectUntilSuccess, waitUntilFilesystemIsOfType } from "./util"; import { apiCall, project } from "@cocalc/api-client"; diff --git a/src/compute/compute/lib/terminal.ts b/src/compute/compute/lib/terminal.ts deleted file mode 100644 index 873432bdb4..0000000000 --- a/src/compute/compute/lib/terminal.ts +++ /dev/null @@ -1,19 +0,0 @@ -/* -Connect from this nodejs process to a remote cocalc project over a websocket and -provide a remote terminal session. -*/ - -import { RemoteTerminal } from "@cocalc/terminal"; - -// path is something like "foo/.bar.term" -export function terminal({ websocket, path, cwd, env, computeServerId }) { - return new RemoteTerminal( - websocket, - path, - { - cwd, - env: { TERM: "screen", ...env }, - }, - computeServerId, - ); -} diff --git a/src/compute/compute/package.json b/src/compute/compute/package.json index 3cfb89b62d..b531d2a4d6 100644 --- a/src/compute/compute/package.json +++ b/src/compute/compute/package.json @@ -8,7 +8,7 @@ "scripts": { "preinstall": "npx only-allow pnpm", "clean": "rm -rf dist node_modules", - "make": "pnpm run build", + "make": "pnpm install && pnpm build", "build": "../../packages/node_modules/.bin/tsc", "tsc": "../../packages/node_modules/.bin/tsc --watch --pretty --preserveWatchOutput" }, @@ -31,13 +31,12 @@ "@cocalc/api-client": "workspace:*", "@cocalc/backend": "workspace:*", "@cocalc/compute": "link:", + "@cocalc/conat": "workspace:*", "@cocalc/jupyter": "workspace:*", - "@cocalc/nats": "workspace:*", "@cocalc/project": "workspace:*", "@cocalc/sync": "workspace:*", "@cocalc/sync-client": "workspace:*", "@cocalc/sync-fs": "workspace:*", - "@cocalc/terminal": "workspace:*", "@cocalc/util": "workspace:*", "@types/ws": "^8.18.1", "awaiting": "^3.0.0", diff --git a/src/compute/compute/tsconfig.json b/src/compute/compute/tsconfig.json index e7dc22d249..7d9f01f99f 100644 --- a/src/compute/compute/tsconfig.json +++ b/src/compute/compute/tsconfig.json @@ -13,7 +13,6 @@ { "path": "../sync" }, { "path": "../sync-client" }, { "path": "../sync-fs" }, - { "path": "../terminal" }, { "path": "../util" } ] } diff --git a/src/compute/conat b/src/compute/conat new file mode 120000 index 0000000000..877aba1148 --- /dev/null +++ b/src/compute/conat @@ -0,0 +1 @@ +../packages/conat \ No newline at end of file diff --git a/src/compute/nats b/src/compute/nats deleted file mode 120000 index 702beda7a3..0000000000 --- a/src/compute/nats +++ /dev/null @@ -1 +0,0 @@ -../packages/nats \ No newline at end of file diff --git a/src/compute/pnpm-lock.yaml b/src/compute/pnpm-lock.yaml index 66e2285c09..66ffcd0b98 100644 --- a/src/compute/pnpm-lock.yaml +++ b/src/compute/pnpm-lock.yaml @@ -17,12 +17,12 @@ importers: '@cocalc/compute': specifier: 'link:' version: 'link:' + '@cocalc/conat': + specifier: workspace:* + version: link:../conat '@cocalc/jupyter': specifier: workspace:* version: link:../jupyter - '@cocalc/nats': - specifier: workspace:* - version: link:../nats '@cocalc/project': specifier: workspace:* version: link:../project @@ -35,9 +35,6 @@ importers: '@cocalc/sync-fs': specifier: workspace:* version: link:../sync-fs - '@cocalc/terminal': - specifier: workspace:* - version: link:../terminal '@cocalc/util': specifier: workspace:* version: link:../util diff --git a/src/compute/pnpm-workspace.yaml b/src/compute/pnpm-workspace.yaml index 62eea2e0eb..5cfb73b0fd 100644 --- a/src/compute/pnpm-workspace.yaml +++ b/src/compute/pnpm-workspace.yaml @@ -3,7 +3,7 @@ packages: - backend - comm - jupyter - - nats + - conat - project - sync - sync-fs @@ -11,7 +11,7 @@ packages: - util - terminal - compute - - nats + - conat onlyBuiltDependencies: - '@cocalc/fuse-native' diff --git a/src/package.json b/src/package.json index 1454fef3fe..00cb8c6363 100644 --- a/src/package.json +++ b/src/package.json @@ -15,19 +15,16 @@ "database": "cd dev/project && ./start_postgres.py", "database-remove-locks": "./scripts/database-remove-locks", "c": "LOGS=/tmp/ DEBUG='cocalc:*' ./scripts/c", - "version-check": "pip3 install typing_extensions mypy || pip3 install --break-system-packages typing_extensions mypy && ./workspaces.py version-check && mypy scripts/check_npm_packages.py", + "version-check": "pip3 install --break-system-packages typing_extensions mypy || pip3 install --break-system-packages typing_extensions mypy && ./workspaces.py version-check && mypy scripts/check_npm_packages.py", "test-parallel": "unset DEBUG && pnpm run version-check && cd packages && pnpm run -r --parallel test", "test": "unset DEBUG && pnpm run version-check && cd packages && pnpm run -r test", + "depcheck": "cd packages && pnpm run -r --parallel depcheck", "prettier-all": "cd packages/", - "nats-server": "cd ${COCALC_ROOT:=$INIT_CWD}/packages/backend && node -e \"require('@cocalc/backend/nats/install').main()\" && node -e \"require('@cocalc/backend/nats/conf').main()\" && node -e \"require('@cocalc/backend/nats/server').main()\"", - "build-nats": "cd ${COCALC_ROOT:=$INIT_CWD}/packages/util && pnpm install && pnpm build && cd ${COCALC_ROOT:=$INIT_CWD}/packages/nats && pnpm install && pnpm build && cd ${COCALC_ROOT:=$INIT_CWD}/packages/backend && pnpm install && pnpm build", - "nats-server-ci": "pnpm run build-nats && cd ${COCALC_ROOT:=$INIT_CWD}/packages/backend && node -e \"require('@cocalc/backend/nats/install').main()\" && node -e \"require('@cocalc/backend/nats/conf').main()\" && node -e \"require('@cocalc/backend/nats/server').main()\"", - "nats-server-daemon": "cd ${COCALC_ROOT:=$INIT_CWD}/packages/backend && node -e \"require('@cocalc/backend/nats/install').main()\" && node -e \"require('@cocalc/backend/nats/conf').main()\" && node -e \"require('@cocalc/backend/nats/server').main({daemon:true})\"", - "nats-server-verbose": "cd ${COCALC_ROOT:=$INIT_CWD}/packages/backend && node -e \"require('@cocalc/backend/nats/install').main()\" && node -e \"require('@cocalc/backend/nats/conf').main()\" && node -e \"require('@cocalc/backend/nats/server').main({verbose:true})\"", - "nats-cli": "cd ${COCALC_ROOT:=$INIT_CWD}/packages/backend && node -e \"require('@cocalc/backend/nats/cli').main()\"", - "nats-sys": "cd ${COCALC_ROOT:=$INIT_CWD}/packages/backend && node -e \"require('@cocalc/backend/nats/cli').main({user:'sys'})\"", - "nats-tiered-storage": "cd ${COCALC_ROOT:=$INIT_CWD}/packages/server && DEBUG=cocalc:* DEBUG_CONSOLE=yes node -e \"require('@cocalc/server/nats/tiered-storage').init()\"", - "local-ci": "./scripts/ci.sh" + "local-ci": "./scripts/ci.sh", + "conat-server": "cd packages/server && pnpm conat-server", + "conat-connections": "cd packages/backend && pnpm conat-connections", + "conat-watch": "cd packages/backend && pnpm conat-watch", + "conat-inventory": "cd packages/backend && pnpm conat-inventory" }, "repository": { "type": "git", diff --git a/src/packages/api-client/package.json b/src/packages/api-client/package.json index dc1615747d..b5d16040ec 100644 --- a/src/packages/api-client/package.json +++ b/src/packages/api-client/package.json @@ -6,23 +6,17 @@ "scripts": { "preinstall": "npx only-allow pnpm", "build": "../node_modules/.bin/tsc --build", - "tsc": "../node_modules/.bin/tsc --watch --pretty --preserveWatchOutput" + "tsc": "../node_modules/.bin/tsc --watch --pretty --preserveWatchOutput", + "depcheck": "pnpx depcheck --ignores @cocalc/api-client " }, - "files": [ - "dist/**", - "bin/**", - "README.md", - "package.json" - ], + "files": ["dist/**", "bin/**", "README.md", "package.json"], "author": "SageMath, Inc.", - "keywords": [ - "cocalc", - "jupyter" - ], + "keywords": ["cocalc", "jupyter"], "license": "SEE LICENSE.md", "dependencies": { "@cocalc/api-client": "workspace:*", - "@cocalc/backend": "workspace:*" + "@cocalc/backend": "workspace:*", + "@cocalc/util": "workspace:*" }, "homepage": "https://github.com/sagemathinc/cocalc/tree/master/src/packages/compute", "repository": { diff --git a/src/packages/api-client/src/project.ts b/src/packages/api-client/src/project.ts index 6fbf6f09f8..e05f0abf97 100644 --- a/src/packages/api-client/src/project.ts +++ b/src/packages/api-client/src/project.ts @@ -5,6 +5,7 @@ API key should be enough to allow them. import { apiCall } from "./call"; import { siteUrl } from "./urls"; +import { type JupyterApiOptions } from "@cocalc/util/jupyter/api-types"; // Starts a project running. export async function start(opts: { project_id: string }) { @@ -67,7 +68,7 @@ export async function readFile(opts: { mesg: { event: "read_file_from_project", ...opts }, }); return siteUrl( - `blobs/${opts.path}${archive ? `.${archive}` : ""}?uuid=${data_uuid}` + `blobs/${opts.path}${archive ? `.${archive}` : ""}?uuid=${data_uuid}`, ); } @@ -101,24 +102,7 @@ export async function readTextFile(opts: { }); } -export async function jupyterExec(opts: { - project_id: string; - hash?: string; // give either hash *or* kernel, input, history, etc. - kernel: string; // jupyter kernel - input: string; // input code to execute - history?: string[]; // optional history of this conversation as a list of input strings. Do not include output - path?: string; // optional path where execution happens - pool?: { size?: number; timeout_s?: number }; - limits?: { - // see packages/jupyter/nbgrader/jupyter-run.ts - timeout_ms_per_cell: number; - max_output_per_cell: number; - max_output: number; - total_output: number; - timeout_ms?: number; - start_time?: number; - }; -}): Promise { +export async function jupyterExec(opts: JupyterApiOptions): Promise { return ( await callProject({ project_id: opts.project_id, diff --git a/src/packages/backend/auth/cookie-names.ts b/src/packages/backend/auth/cookie-names.ts index 9734372be1..6ba65bd330 100644 --- a/src/packages/backend/auth/cookie-names.ts +++ b/src/packages/backend/auth/cookie-names.ts @@ -39,3 +39,18 @@ export const ACCOUNT_ID_COOKIE_NAME = basePathCookieName({ basePath, name: "account_id", }); + +export const PROJECT_SECRET_COOKIE_NAME = basePathCookieName({ + basePath, + name: "project_secret", +}); + +export const PROJECT_ID_COOKIE_NAME = basePathCookieName({ + basePath, + name: "project_id", +}); + +export const HUB_PASSWORD_COOKIE_NAME = basePathCookieName({ + basePath, + name: "hub_password", +}); diff --git a/src/packages/backend/bash.test.ts b/src/packages/backend/bash.test.ts new file mode 100644 index 0000000000..6f89fe453b --- /dev/null +++ b/src/packages/backend/bash.test.ts @@ -0,0 +1,39 @@ +import bash from "@cocalc/backend/bash"; +import { once } from "@cocalc/util/async-utils"; + +describe("test the bash child process spawner", () => { + it("echos 'hi'", async () => { + const child = await bash("echo 'hi'"); + let out = ""; + child.stdout.on("data", (data) => { + out += data.toString(); + }); + await once(child, "exit"); + expect(out).toBe("hi\n"); + }); + + it("runs a multiline bash script", async () => { + const child = await bash(` +sum=0 +for i in {1..100}; do + sum=$((sum + i)) +done +echo $sum +`); + let out = ""; + child.stdout.on("data", (data) => { + out += data.toString(); + }); + await once(child, "exit"); + expect(out).toBe("5050\n"); + }); + + it("runs a bash script with ulimit to limit execution time", async () => { + const child = await bash(` +ulimit -t 1 +while : ; do : ; done # infinite CPU +`); + const x = await once(child, "exit"); + expect(x[1]).toBe("SIGKILL"); + }); +}); diff --git a/src/packages/backend/bash.ts b/src/packages/backend/bash.ts new file mode 100644 index 0000000000..86d1245628 --- /dev/null +++ b/src/packages/backend/bash.ts @@ -0,0 +1,36 @@ +/* +Easily spawn a bash script given by a string, which is written to a temp file +that is automatically removed on exit. Returns a child process. +*/ + +import { chmod, mkdtemp, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { ChildProcessWithoutNullStreams, spawn } from "node:child_process"; +import { cleanUpTempDir } from "./execute-code"; +import { join } from "node:path"; +//import getLogger from "@cocalc/backend/logger"; +//const logger = getLogger("bash"); + +export default async function bash( + command: string, + spawnOptions?, +): Promise { + let tempDir = ""; + let tempPath = ""; + try { + tempDir = await mkdtemp(join(tmpdir(), "cocalc-")); + tempPath = join(tempDir, "a.sh"); + //logger.debug("bash:writing temp file that contains bash program:\n", command); + await writeFile(tempPath, command); + await chmod(tempPath, 0o700); + } catch (err) { + await cleanUpTempDir(tempDir); + throw err; + } + //logger.debug(`spawning bash program: ${JSON.stringify(command).slice(0,1000)}...`); + const child = spawn("bash", [tempPath], spawnOptions); + child.once("exit", async () => { + await cleanUpTempDir(tempDir); + }); + return child; +} diff --git a/src/packages/backend/bin/conat-connections.cjs b/src/packages/backend/bin/conat-connections.cjs new file mode 100755 index 0000000000..699e62f9ad --- /dev/null +++ b/src/packages/backend/bin/conat-connections.cjs @@ -0,0 +1,17 @@ +const { conat } = require('@cocalc/backend/conat') +const { showUsersAndStats } = require('@cocalc/conat/monitor/tables'); +const { conatServer } = require('@cocalc/backend/data') + +async function main() { + console.log("Connecting to", conatServer); + const maxMessages = process.argv[2] ? parseInt(process.argv[2]) : undefined; + const maxWait = process.argv[3] ? parseInt(process.argv[3]) : 3000; + const client = conat(); + if(!maxMessages) { + console.log("\nUsage: pnpm conat-connnections [num-servers] [max-wait-ms]\n") + } + await showUsersAndStats({client, maxWait, maxMessages}); + process.exit(0); +} + +main(); diff --git a/src/packages/backend/bin/conat-disconnect.cjs b/src/packages/backend/bin/conat-disconnect.cjs new file mode 100755 index 0000000000..f1227a4ed3 --- /dev/null +++ b/src/packages/backend/bin/conat-disconnect.cjs @@ -0,0 +1,13 @@ +const { conat } = require('@cocalc/backend/conat') +const { connections } = require('@cocalc/conat/monitor/tables'); + +async function main() { + console.log("Disconnect Clients From Server") + const ids = process.argv.slice(2); + console.log(ids); + const client = await conat() + await client.call('sys.conat.server').disconnect(ids); + process.exit(0); +} + +main(); diff --git a/src/packages/backend/bin/conat-inventory.cjs b/src/packages/backend/bin/conat-inventory.cjs new file mode 100755 index 0000000000..1ed4fee128 --- /dev/null +++ b/src/packages/backend/bin/conat-inventory.cjs @@ -0,0 +1,17 @@ +require('@cocalc/backend/conat') +const { inventory } = require('@cocalc/backend/conat/sync') + +async function main() { + console.log("\n\nUsage: pnpm conat-inventory project_id [filter] --notrunc\n\n") + const project_id = process.argv[2]; + if(!project_id) { + process.exit(1); + } + const filter = process.argv[3]; + const noTrunc = !!process.argv[4] + const i = await inventory({project_id}) + await i.ls({filter, noHelp:true, noTrunc}); + process.exit(0); +} + +main(); diff --git a/src/packages/backend/bin/conat-persist.cjs b/src/packages/backend/bin/conat-persist.cjs new file mode 100755 index 0000000000..cf59952fc0 --- /dev/null +++ b/src/packages/backend/bin/conat-persist.cjs @@ -0,0 +1,14 @@ +/* +run a persist server +*/ + +const { conat } = require('@cocalc/backend/conat') +require('@cocalc/backend/conat/persist'); +const {server} = require('@cocalc/conat/persist/server'); + +async function main() { + const client = await conat() + server({client}); +} + +main(); diff --git a/src/packages/backend/bin/conat-watch.cjs b/src/packages/backend/bin/conat-watch.cjs new file mode 100755 index 0000000000..5a41013263 --- /dev/null +++ b/src/packages/backend/bin/conat-watch.cjs @@ -0,0 +1,10 @@ +const { conat } = require('@cocalc/backend/conat') + +async function main() { + const subject = process.argv[2] ?? '>'; + console.log("watching ", {subject}) + const cn = await conat() + cn.watch(subject) +} + +main(); diff --git a/src/packages/backend/conat/conat.ts b/src/packages/backend/conat/conat.ts new file mode 100644 index 0000000000..1d875e8d11 --- /dev/null +++ b/src/packages/backend/conat/conat.ts @@ -0,0 +1,17 @@ +import { conatPassword, conatServer } from "@cocalc/backend/data"; +import { connect, Client, type ClientOptions } from "@cocalc/conat/core/client"; +import { HUB_PASSWORD_COOKIE_NAME } from "@cocalc/backend/auth/cookie-names"; +import { inboxPrefix } from "@cocalc/conat/names"; + +export type { Client }; + +export function conat(options?: ClientOptions): Client { + return connect({ + address: conatServer, + inboxPrefix: inboxPrefix({ hub_id: "hub" }), + extraHeaders: { + Cookie: `${HUB_PASSWORD_COOKIE_NAME}=${conatPassword}`, + }, + ...options, + }); +} diff --git a/src/packages/backend/conat/index.ts b/src/packages/backend/conat/index.ts new file mode 100644 index 0000000000..c59b328d97 --- /dev/null +++ b/src/packages/backend/conat/index.ts @@ -0,0 +1,10 @@ +import getLogger from "@cocalc/backend/logger"; +import { setConatClient } from "@cocalc/conat/client"; +import { conat } from "./conat"; + +export { conat }; + +export function init() { + setConatClient({ conat, getLogger }); +} +init(); diff --git a/src/packages/backend/conat/persist.ts b/src/packages/backend/conat/persist.ts new file mode 100644 index 0000000000..c30915d13b --- /dev/null +++ b/src/packages/backend/conat/persist.ts @@ -0,0 +1,45 @@ +/* + +To test having multiple persist servers at once in dev mode, start +up your dev server. Then do the following in nodejs to create an +additional persist server: + + require("@cocalc/backend/conat/persist").initPersistServer() + +*/ + +import "./index"; +import betterSqlite3 from "better-sqlite3"; +import { initContext } from "@cocalc/conat/persist/context"; +import { compress, decompress } from "zstd-napi"; +import { syncFiles } from "@cocalc/backend/data"; +import ensureContainingDirectoryExists from "@cocalc/backend/misc/ensure-containing-directory-exists"; + +initContext({ + betterSqlite3, + compress, + decompress, + syncFiles, + ensureContainingDirectoryExists, +}); + +export { pstream } from "@cocalc/conat/persist/storage"; +import { server } from "@cocalc/conat/persist/server"; +export { server }; +import { conat } from "./conat"; + +const persistServers: any[] = []; + +export function initPersistServer() { + const persistServer = server({ + client: conat({ noCache: persistServers.length > 0 }), + }); + persistServers.push(persistServer); +} + +export function close() { + for (const persistServer of persistServers) { + persistServer.end(); // end is a bit more graceful + } + persistServers.length = 0; +} diff --git a/src/packages/backend/conat/sync.ts b/src/packages/backend/conat/sync.ts new file mode 100644 index 0000000000..3bccd54978 --- /dev/null +++ b/src/packages/backend/conat/sync.ts @@ -0,0 +1,40 @@ +import { + dstream as createDstream, + type DStream, + type DStreamOptions as DstreamCreateOptions, +} from "@cocalc/conat/sync/dstream"; +import { dkv as createDKV, type DKV, type DKVOptions } from "@cocalc/conat/sync/dkv"; +import { dko as createDKO, type DKO } from "@cocalc/conat/sync/dko"; +import { akv as createAKV, type AKV } from "@cocalc/conat/sync/akv"; +import { astream as createAStream, type AStream } from "@cocalc/conat/sync/astream"; +import { createOpenFiles, type OpenFiles } from "@cocalc/conat/sync/open-files"; +export { inventory } from "@cocalc/conat/sync/inventory"; +import "./index"; + +export type { DStream, DKV, DKO, AKV }; + +export async function dstream( + opts: DstreamCreateOptions, +): Promise> { + return await createDstream(opts); +} + +export function astream(opts: DstreamCreateOptions): AStream { + return createAStream(opts); +} + +export async function dkv(opts: DKVOptions): Promise> { + return await createDKV(opts); +} + +export function akv(opts: DKVOptions): AKV { + return createAKV(opts); +} + +export async function dko(opts: DKVOptions): Promise> { + return await createDKO(opts); +} + +export async function openFiles(project_id: string, opts?): Promise { + return await createOpenFiles({ project_id, ...opts }); +} diff --git a/src/packages/backend/conat/test/core/basic.test.ts b/src/packages/backend/conat/test/core/basic.test.ts new file mode 100644 index 0000000000..a21d8db409 --- /dev/null +++ b/src/packages/backend/conat/test/core/basic.test.ts @@ -0,0 +1,372 @@ +/* +Very basic test of conat core client and server. + +pnpm test `pwd`/basic.test.ts + + +*/ + +import { connect, before, after, wait } from "@cocalc/backend/conat/test/setup"; + +beforeAll(before); + +describe("connect to the server from a client", () => { + it("creates a client and confirm it connects", async () => { + const cn = connect(); + expect(cn.conn.connected).toBe(false); + await cn.waitUntilConnected(); + expect(cn.conn.connected).toBe(true); + cn.close(); + expect(cn.conn.connected).toBe(false); + }); + + it("creates a client and waits for the info field to get set", async () => { + const cn = connect(); + await wait({ until: () => cn.info != null }); + expect(cn.info?.max_payload).toBeGreaterThan(10000); + }); +}); + +describe("basic test of publish and subscribe", () => { + let sub; + + let subject = "conat"; + let cn, cn2; + it("creates a subscription to 'conat'", async () => { + cn = connect(); + sub = await cn.subscribe(subject); + }); + + it("publishes to 'conat' and verifies that the subscription receives the message", async () => { + const data = "cocalc"; + await cn.publish(subject, data); + const { value, done } = await sub.next(); + expect(value.data).toEqual(data); + expect(done).toBe(false); + }); + + it("publishes using a second client", async () => { + const data = null; + cn2 = connect(); + expect(cn === cn2).toEqual(false); + await cn2.publish(subject, data); + const { value } = await sub.next(); + expect(value.data).toEqual(data); + }); + + const count = 15; + + it(`publish ${count} messages and confirm receipt via sub.next`, async () => { + for (let i = 0; i < count; i++) { + cn.publish(subject, i); + } + for (let i = 0; i < count; i++) { + const { value } = await sub.next(); + expect(value.data).toBe(i); + } + }); + + it(`publish ${count} messages and confirm receipt via async iteration`, async () => { + const w: number[] = []; + for (let i = 0; i < count; i++) { + cn.publish(subject, i); + w.push(i); + } + const v: number[] = []; + for await (const x of sub) { + v.push(x.data); + if (v.length == w.length) { + break; + } + } + expect(w).toEqual(v); + }); + + it("confirm exiting the async iterator above using break ended the subscription", async () => { + // this is how async iterators work... + const { done } = await sub.next(); + expect(done).toBe(true); + expect(sub.ended).toBe(true); + }); + + it("we can now make a new subscription to the same subject. We then stop the subscription and confirm it ends", async () => { + const sub2 = await cn.subscribe(subject); + sub2.stop(); + const { value, done } = await sub.next(); + expect(value).toBe(undefined); + expect(done).toBe(true); + }); + + it("verify that you can't subscribe twice to the same subject with a single client but *different* queue groups", async () => { + const sub1 = await cn.subscribe(subject); + await expect(async () => { + await cn.subscribe(subject, { queue: "xxx" }); + }).rejects.toThrowError("one queue group"); + + sub1.stop(); + // now this works + const sub2 = await cn.subscribe(subject); + sub2.stop(); + }); + + const subject2 = "foo.*.bar.>"; + it(`tests using the subject '${subject2}' with a wildcard and >`, async () => { + const sub = await cn.subscribe(subject2); + // this is ignored + cn.publish("foo.x", "abc"); + // this is received + cn.publish("foo.a.bar.b", "xxx", { headers: { a: "b" } }); + const { value: mesg } = await sub.next(); + expect(mesg.data).toBe("xxx"); + expect(mesg.headers).toEqual({ a: "b" }); + expect(mesg.subject).toBe("foo.a.bar.b"); + }); + + it("queue groups -- same queue groups, so exactly one gets the message", async () => { + const sub1 = await cn.subscribe("pub", { queue: "1" }); + const sub2 = await cn2.subscribe("pub", { queue: "1" }); + const { count } = await cn.publish("pub", "hello"); + expect(count).toBe(1); + let count1 = 0; + let count2 = 0; + (async () => { + await sub1.next(); + count1 += 1; + })(); + (async () => { + await sub2.next(); + count2 += 1; + })(); + await wait({ until: () => count1 + count2 > 0 }); + expect(count1 + count2).toBe(1); + sub1.stop(); + sub2.stop(); + }); + + it("queue groups -- distinct queue groups ALL get the message", async () => { + const sub1 = await cn.subscribe("pub3", { queue: "1" }); + const sub2 = await cn2.subscribe("pub3", { queue: "2" }); + const { count } = await cn.publish("pub3", "hello"); + expect(count).toBe(2); + const { value: mesg1 } = await sub1.next(); + const { value: mesg2 } = await sub2.next(); + expect(mesg1.data).toBe("hello"); + expect(mesg2.data).toBe("hello"); + }); +}); + +describe("basic tests of request/respond", () => { + let c1, c2; + + it("create two clients", () => { + c1 = connect(); + c2 = connect(); + }); + + let sub; + it("make one client be an eval server", async () => { + sub = await c2.subscribe("eval"); + (async () => { + for await (const mesg of sub) { + mesg.respond(eval(mesg.data)); + } + })(); + }); + + it("send a request and gets a response", async () => { + const resp = await c1.request("eval", "1+2+3+4+5+6+7+8+9+10"); + expect(resp.data).toBe(55); + }); + + it("'server' can also send a request and gets a response", async () => { + const resp = await c2.request("eval", "1+2+3+4+5"); + expect(resp.data).toBe(15); + }); + + it("send a request to a server that doesn't exist and get 503 error", async () => { + try { + await c2.request("does-not-exist", "1+2+3+4+5"); + } catch (err) { + expect(err.code == 503); + } + }); + + it("stop our server above (close subscription) and confirm get 503 error", async () => { + const n = c2.numSubscriptions(); + sub.close(); + expect(c2.numSubscriptions()).toBe(n - 1); + await c2.syncSubscriptions(); + expect(c2.numSubscriptions()).toBe(n - 1); + try { + await c1.request("eval", "1+2+3+4+5", { timeout: 2000 }); + } catch (err) { + expect(err.code).toBe(503); + } + }); + + let callIter; + it("create a requestMany server that iterates over what you send it", async () => { + // This example illustrates how to define a requestMany server + // and includes error handling. Note the technique of using + // the *headers* for control signalling (e.g., when we're done, or if + // there is an error) and using the message payload for the actual data. + // In Conat headers are very well supported, encouraged, and easy to use + // (and arbitrary JSON), unlike NATS.js. + + sub = await c2.subscribe("iter"); + (async () => { + for await (const mesg of sub) { + try { + for (const x of mesg.data) { + mesg.respond(x, { headers: { done: false } }); + } + mesg.respond(null, { headers: { done: true } }); + } catch (err) { + mesg.respond(null, { headers: { done: true, error: `${err}` } }); + return; + } + } + })(); + + // also function to do request + callIter = async (client, x) => { + const iter = await client.requestMany("iter", x); + const v: any[] = []; + for await (const resp of iter) { + if (resp.headers?.error) { + throw Error(resp.headers?.error); + } + if (resp.headers.done) { + return v; + } + v.push(resp.data); + } + return v; + }; + }); + + it("call the iter server -- a simple test", async () => { + const w = [3, 8, 9]; + const v = await callIter(c1, w); + expect(v).toEqual(w); + expect(v).not.toBe(w); + + // also from other client + const v2 = await callIter(c2, w); + expect(v2).toEqual(w); + }); + + it("call the iter server -- test that throws an error", async () => { + await expect(async () => { + await callIter(c1, null); + }).rejects.toThrowError("is not iterable"); + }); +}); + +describe("creating multiple subscriptions to the same subject", () => { + let subject = "conat"; + let s1, s2; + let c1, c2; + it("creates clients and two subscriptions to same subject using the same client", async () => { + c1 = connect(); + c2 = connect(); + s1 = await c1.subscribe(subject); + s2 = await c1.subscribe(subject); + expect(s1 === s2).toBe(false); + }); + + it("publishes to 'conat' and verifies that each subscription indendently receives each message", async () => { + const data = "cocalc"; + await c2.publish(subject, data); + const { value, done } = await s1.next(); + expect(value.data).toEqual(data); + expect(done).toBe(false); + const { value: value2, done: done2 } = await s2.next(); + expect(value2.data).toEqual(data); + expect(done2).toBe(false); + + c2.publish(subject, 1); + c2.publish(subject, 2); + c2.publish(subject, 3); + expect((await s1.next()).value.data).toBe(1); + expect((await s2.next()).value.data).toBe(1); + expect((await s1.next()).value.data).toBe(2); + expect((await s1.next()).value.data).toBe(3); + expect((await s2.next()).value.data).toBe(2); + expect((await s2.next()).value.data).toBe(3); + }); + + it("closing properly reference counts", async () => { + expect(c1.subs[subject].refCount).toBe(2); + s1.close(); + expect(c1.subs[subject].refCount).toBe(1); + expect(c1.queueGroups[subject] == null).toBe(false); + c2.publish(subject, 4); + expect((await s2.next()).value.data).toBe(4); + s2.close(); + expect(c1.subs[subject]).toBe(undefined); + expect(c1.queueGroups[subject]).toBe(undefined); + }); + + it("sync subscriptions also work", async () => { + s1 = c1.subscribeSync(subject); + s2 = c1.subscribeSync(subject); + expect(s1 === s2).toBe(false); + await c2.publish(subject, 5); + expect((await s1.next()).value.data).toBe(5); + expect((await s2.next()).value.data).toBe(5); + s1.close(); + s2.close(); + expect(c1.subs[subject]).toBe(undefined); + }); +}); + +const requestManyCount = 3; +describe(`use requestMany to send one message and receive responses from ${requestManyCount} servers in parallel`, () => { + let subject = "requst.many.parallel"; + let subs: any[] = []; + let clients: any[] = []; + + it(`creates ${requestManyCount} subscribers`, async () => { + for (let i = 0; i < requestManyCount; i++) { + const client = connect(); + // use a different queue group for each: + const sub = await client.subscribe(subject, { queue: `${i}` }); + sub.emitter.on("message", (mesg) => { + mesg.respondSync(i); + }); + clients.push(client); + subs.push(sub); + } + }); + + it(`uses requestMany to call all ${requestManyCount} subscribers in parallel and get **all** ${requestManyCount} results that come back within 500ms`, async () => { + const client = connect(); + const responses: any[] = []; + for await (const x of await client.requestMany(subject, "hello", { + timeout: 500, + })) { + responses.push(x.data); + if (responses.length == requestManyCount) { + break; + } + } + expect(new Set(responses).size).toBe(requestManyCount); + client.close(); + }); + + it(`use request to call all ${requestManyCount} subscribers in parallel and get back the first to respond (all else are ignored)`, async () => { + const client = connect(); + const resp = await client.request(subject, "hello"); + expect(typeof resp.data).toBe("number"); + client.close(); + }); + + it("cleans up", () => { + for (const client of clients) { + client.close(); + } + }); +}); + +afterAll(after); diff --git a/src/packages/backend/conat/test/core/connect.test.ts b/src/packages/backend/conat/test/core/connect.test.ts new file mode 100644 index 0000000000..501894aca8 --- /dev/null +++ b/src/packages/backend/conat/test/core/connect.test.ts @@ -0,0 +1,179 @@ +/* + +pnpm test ./connect.test.ts + +*/ + +import getPort from "@cocalc/backend/get-port"; +import { + before, + after, + restartServer, + connect, + initConatServer, +} from "@cocalc/backend/conat/test/setup"; +import { delay } from "awaiting"; +import { wait } from "@cocalc/backend/conat/test/util"; +import { once } from "@cocalc/util/async-utils"; + +let port; +beforeAll(async () => { + await before(); + port = await getPort(); +}); + +describe("basic test of restarting the server causing a reconnect of client", () => { + let cn; + it("starts a client connecting to that port, despite there being no server yet", async () => { + cn = connect(); + expect(cn.conn.connected).toBe(false); + expect(cn.state).toBe("disconnected"); + }); + + it("now client should connect", async () => { + const connected = once(cn, "connected"); + await cn.waitUntilConnected(); + expect(cn.conn.connected).toBe(true); + await connected; // verify connected event fired + expect(cn.state).toBe("connected"); + }); + + it("close server and observe client disconnects, then connects again", async () => { + expect(cn.state).toBe("connected"); + restartServer(); + await once(cn, "disconnected"); + await once(cn, "connected"); + }); + + it("clean up", () => { + cn.close(); + }); +}); +describe("create server *after* client and ensure connects properly", () => { + let cn; + it("starts a client connecting to that port, despite there being no server yet", async () => { + cn = connect({ + address: `http://localhost:${port}`, + reconnectionDelay: 25, // fast for tests + randomizationFactor: 0, + }); + await delay(20); + expect(cn.conn.connected).toBe(false); + expect(cn.state).toBe("disconnected"); + }); + + let server; + it("create a server", async () => { + server = await initConatServer({ port }); + }); + + it("now client should connect", async () => { + const connected = once(cn, "connected"); + await cn.waitUntilConnected(); + expect(cn.conn.connected).toBe(true); + await connected; // verify connected event fired + }); + + it("close server and observe client disconnect", async () => { + const disconnected = once(cn, "disconnected"); + server.close(); + await wait({ until: () => !cn.conn.connected }); + expect(cn.conn.connected).toBe(false); + await disconnected; // verify disconnected event fired + }); + + it("create server again and observe client connects again", async () => { + const connected = once(cn, "connected"); + server = await initConatServer({ port }); + await wait({ until: () => cn.conn.connected }); + expect(cn.conn.connected).toBe(true); + await connected; // verify connected event fired + }); + + it("clean up", () => { + server.close(); + cn.close(); + }); +}); + +describe("create server after sync creating a subscription and publishing a message, and observe that messages are dropped", () => { + // The moral here is do NOT use subscribeSync and publishSync + // unless you don't care very much... + let cn; + it("starts a client, despite there being no server yet", async () => { + cn = connect({ address: `http://localhost:${port}` }); + expect(cn.conn.connected).toBe(false); + }); + + let sub; + it("create a subscription before the server exists", () => { + sub = cn.subscribeSync("xyz"); + const { bytes } = cn.publishSync("xyz", "hello"); + expect(bytes).toBe(6); + cn.publishSync("xyz", "conat"); + }); + + let server; + it("start the server", async () => { + server = await initConatServer({ port }); + await wait({ until: () => cn.conn.connected }); + await delay(50); + }); + + it("see that both messages we sent before connecting were dropped", async () => { + const { bytes, count } = await cn.publish("xyz", "more"); + expect(count).toBe(1); + expect(bytes).toBe(5); + const { value: mesg1 } = await sub.next(); + // we just got a message but it's AFTER the two above. + expect(mesg1.data).toBe("more"); + }); + + it("clean up", () => { + server.close(); + cn.close(); + sub.close(); + }); +}); + +describe("create server after async creating a subscription and async publishing a message, and observe that it DOES works", () => { + let cn; + it("starts a client, despite there being no server yet", async () => { + cn = connect({ address: `http://localhost:${port}` }); + expect(cn.conn.connected).toBe(false); + }); + + let sub; + let recv: any[] = []; + it("create a sync subscription before the server exists", () => { + const f = async () => { + sub = await cn.subscribe("xyz"); + await cn.publish("xyz", "hello"); + const { value: mesg } = await sub.next(); + recv.push(mesg.data); + await cn.publish("xyz", "conat"); + const { value: mesg2 } = await sub.next(); + recv.push(mesg2.data); + }; + f(); + }); + + let server; + it("start the server", async () => { + server = await initConatServer({ port }); + await wait({ until: () => cn.conn.connected }); + }); + + it("see that both messages we sent before connecting arrive", async () => { + await wait({ until: () => recv.length == 2 }); + expect(recv).toEqual(["hello", "conat"]); + }); + + it("clean up", () => { + server.close(); + cn.close(); + sub.close(); + }); +}); + +afterAll(after); diff --git a/src/packages/backend/conat/test/core/core-stream-break.test.ts b/src/packages/backend/conat/test/core/core-stream-break.test.ts new file mode 100644 index 0000000000..0be45d5042 --- /dev/null +++ b/src/packages/backend/conat/test/core/core-stream-break.test.ts @@ -0,0 +1,116 @@ +/* +Testing that core-stream works even with attempts to break it: + +- by stopping/starting the persist server at key moments. + +pnpm test ./core-stream-break.test.ts + +*/ + +import { server as initPersistServer } from "@cocalc/backend/conat/persist"; +import { dstream } from "@cocalc/backend/conat/sync"; +import { cstream } from "@cocalc/conat/sync/core-stream"; +import { + connect, + before, + after, + wait, + delay, + persistServer as setupPersistServer, + setDefaultTimeouts, +} from "@cocalc/backend/conat/test/setup"; + +beforeAll(before); + +describe("stop persist server, create a client, create an ephemeral core-stream, then start persist server", () => { + let client; + let stream; + let pclient; + let persistServer; + + // use much shorter timeout for this test of restarting persist server, which isn't a NORMAL thing. + setDefaultTimeouts({ request: 750, publish: 750 }); + + it("close the persist server that was setup as part of before above", async () => { + await setupPersistServer.end(); + }); + + it("start persist server, then create ephemeral core stream (verifying that persist server can be stopped then started and it works)", async () => { + pclient = connect(); + client = connect(); + persistServer = initPersistServer({ client: pclient }); + stream = await cstream({ client, name: "test1" }); + expect(stream.length).toBe(0); + await stream.publish("x"); + stream.close(); + }); + + it("stops persist server again, but this time starts creating csteam before starting persist server", async () => { + await persistServer.end({ timeout: 500 }); + stream = null; + // start creating the cstream immediately: + (async () => { + stream = await cstream({ client, name: "test1" }); + })(); + // wait before starting the persist server + await delay(100); + persistServer = initPersistServer({ client: pclient }); + await wait({ until: () => stream != null }); + expect(stream.length).toBe(1); + expect(stream.get(0)).toBe("x"); + }); + + it("stops persist server again, and sees that publishing throws timeout error (otherwise it queues things up waiting for persist server to return)", async () => { + await persistServer.end(); + + await expect(async () => { + await stream.publish("y", { timeout: 100 }); + }).rejects.toThrowError(); + + try { + await stream.publish("y", { timeout: 100 }); + } catch (err) { + expect(`${err}`).toContain("timeout"); + } + }); + + it("starts persist server and can eventually publish again", async () => { + persistServer = initPersistServer({ client: pclient }); + await wait({ + until: async () => { + try { + await stream.publish("y"); + return true; + } catch (err) { + return false; + } + }, + }); + }); + + it("creates a dstream, publishes, sees it hasn't saved, starts persist server and sees save works again", async () => { + const d = await dstream({ name: "test2" }); + await persistServer.end({ timeout: 500 }); + d.publish("x"); + expect(d.hasUnsavedChanges()).toBe(true); + persistServer = initPersistServer({ client: pclient }); + await d.save(); + expect(d.hasUnsavedChanges()).toBe(false); + d.close(); + }); + + it("terminates persist server and verifies can still creates a dstream, after starting the persist server", async () => { + await persistServer.end(); + let d; + stream = null; + (async () => { + d = await dstream({ client, name: "test2" }); + })(); + persistServer = initPersistServer({ client: pclient }); + await wait({ until: () => d != null }); + expect(d.get(0)).toBe("x"); + persistServer.close(); + }); +}); + +afterAll(after); diff --git a/src/packages/backend/conat/test/core/core-stream.test.ts b/src/packages/backend/conat/test/core/core-stream.test.ts new file mode 100644 index 0000000000..df2b7637d1 --- /dev/null +++ b/src/packages/backend/conat/test/core/core-stream.test.ts @@ -0,0 +1,479 @@ +/* +DEVELOPMENT: + + +pnpm test ./core-stream.test.ts + +*/ + +import { connect, before, after } from "@cocalc/backend/conat/test/setup"; +import { + cstream, + KEY_GC_THRESH, + CoreStream, +} from "@cocalc/conat/sync/core-stream"; +import { wait } from "@cocalc/backend/conat/test/util"; +import { is_date as isDate } from "@cocalc/util/misc"; +import { delay } from "awaiting"; +import { once } from "@cocalc/util/async-utils"; + +beforeAll(before); + +describe("create a client, create an ephemeral core-stream, and do basic tests", () => { + let client; + let stream; + let name = `test-${Math.random()}`; + const opts = { + name, + ephemeral: true, + noCache: true, + }; + + it("creates ephemeral core stream", async () => { + client = connect(); + stream = await cstream({ client, ...opts }); + expect(stream.length).toBe(0); + }); + + it("publish some messages", async () => { + // publish null + await stream.publish(null); + await wait({ until: () => stream.length == 1 }); + expect(stream.get(0)).toBe(null); + expect(stream.length).toBe(1); + + // publish a Buffer stays a Buffer + await stream.publish(Buffer.from("xyz")); + await wait({ until: () => stream.length == 2 }); + expect(stream.get(1)).toEqual(Buffer.from("xyz")); + expect(Buffer.isBuffer(stream.get(1))).toBe(true); + expect(stream.length).toBe(2); + + // publish a Date stays a Date + const now = new Date(); + await stream.publish(now); + await wait({ until: () => stream.get(2) != null }); + expect(stream.get(2)).toEqual(now); + expect(isDate(stream.get(2))).toEqual(true); + }); + + it("publishing undefined is not allowed", async () => { + await expect( + async () => await stream.publish(undefined), + ).rejects.toThrowError("must not be 'undefined'"); + }); + + it("a second client has the same messages", async () => { + const client2 = connect(); + const stream2 = await cstream({ + client: client2, + ...opts, + }); + await wait({ until: () => stream2.length == 3 }); + expect(stream2.getAll()).toEqual(stream.getAll()); + stream2.close(); + }); + + it("close and create and see that it's ephemeral", async () => { + // with heavy parallel load when testing a lot at once, this + // can take more than one try: + await wait({ + until: async () => { + stream.close(); + await delay(2500); + stream = await cstream({ client, ...opts }); + return stream.length == 0; + }, + }); + expect(stream.length).toBe(0); + }); + + const count = 100; + it(`publish ${count} messages and observe it works`, async () => { + const v: number[] = []; + for (let i = 0; i < 100; i++) { + await stream.publish(i); + v.push(i); + } + await wait({ until: () => stream.length == 100 }); + expect(stream.length).toBe(100); + expect(stream.getAll()).toEqual(v); + }); + + it("publish a message with a header", async () => { + const n = stream.length; + await stream.publish("body", { headers: { foo: { 10: 5 } } }); + await wait({ until: () => stream.length > n }); + const headers = stream.headers(stream.length - 1); + expect(headers).toEqual(expect.objectContaining({ foo: { 10: 5 } })); + }); + + it("some time consistency checks", () => { + expect( + Math.abs(stream.time(stream.length - 1).valueOf() - Date.now()), + ).toBeLessThan(100); + const times = stream.times(); + expect(times.length).toBe(stream.length); + expect(times.slice(-1)[0]).toEqual(stream.time(stream.length - 1)); + }); + + it("stats consistency check", () => { + const stats = stream.stats(); + expect(stats.count).toBe(stream.length); + expect(stats.bytes).not.toBeNaN(); + expect(stats.bytes).toBeGreaterThan(100); + }); + + it("delete everything in the stream", async () => { + await stream.delete({ all: true }); + await wait({ until: () => stream.length == 0 }); + expect(stream.length).toBe(0); + const stats = stream.stats(); + expect(stats.count).toBe(0); + expect(stats.bytes).toBe(0); + }); + + it("clean up", () => { + stream.close(); + client.close(); + }); +}); + +describe("test basic key:value functionality for persistent core stream", () => { + let client; + let stream; + let name = "kv0"; + + it("creates persistent core stream", async () => { + client = connect(); + stream = await cstream({ client, name, ephemeral: false }); + expect(stream.length).toBe(0); + expect(stream.start_seq).toBe(undefined); + }); + + let seq; + + it("writes a key:value and confirms it was written", async () => { + await stream.setKv("key", "value"); + expect(await stream.getKv("key")).toEqual("value"); + seq = stream.seqKv("key"); + }); + + it("also confirm via getAllKv", () => { + expect(stream.getAllKv()).toEqual({ key: "value" }); + }); + + it("closes and reopens stream, to confirm the key was persisted", async () => { + await stream.close(); + expect(stream.kv).toBe(undefined); + stream = await cstream({ client, name, ephemeral: false }); + expect(stream.hasKv("key")).toBe(true); + expect(stream.hasKv("key2")).toBe(false); + expect(stream.length).toBe(1); + expect(await stream.getKv("key")).toEqual("value"); + expect(stream.seqKv("key")).toBe(seq); + }); + + let client2; + let stream2; + it("create a second client and observe it sees the correct value", async () => { + client2 = connect(); + stream2 = await cstream({ + client: client2, + name, + ephemeral: false, + noCache: true, + }); + expect(await stream2.getKv("key")).toEqual("value"); + }); + + it("modify the value via the second client and see it change in the first", async () => { + await stream2.setKv("key", "value2"); + await wait({ until: () => stream.getKv("key") == "value2" }); + }); + + it("verify that the overwritten message is cleared to save space in both streams", () => { + expect(stream.get(0)).not.toBe(undefined); + expect(stream2.get(0)).not.toBe(undefined); + stream.gcKv(); + stream2.gcKv(); + expect(stream.get(0)).toBe(undefined); + expect(stream2.get(0)).toBe(undefined); + expect(stream.headers(0)).toBe(undefined); + expect(stream2.headers(0)).toBe(undefined); + }); + + it("write a large key:value, then write it again to cause automatic garbage collection", async () => { + await stream.setKv("key", Buffer.from("x".repeat(KEY_GC_THRESH + 10))); + expect(stream.get(stream.length - 1).length).toBe(KEY_GC_THRESH + 10); + await stream.setKv("key", Buffer.from("x".repeat(KEY_GC_THRESH + 10))); + // it's gone + expect(stream.get(stream.length - 2)).toBe(undefined); + }); + + it("close and reload and note there is only one item in the stream (the first message was removed since it is no longer needed)", async () => { + await stream.close(); + expect(stream.kv).toBe(undefined); + stream = await cstream({ client, name, ephemeral: false }); + expect(stream.length).toBe(1); + expect(stream.seqKv(0)).toBe(stream2.seqKv(1)); + }); + + it("cleans up", () => { + stream.close(); + stream2.close(); + client.close(); + client2.close(); + }); +}); + +describe("test key:value delete", () => { + let client; + let stream; + let name = "kvd"; + let client2; + let stream2; + + it("creates new persistent core stream with two copies/clients", async () => { + client = connect(); + stream = await cstream({ client, name, ephemeral: false }); + + client2 = connect(); + stream2 = await cstream({ + client: client2, + name, + ephemeral: false, + noCache: true, + }); + }); + + it("writes to key:value and confirms it was written", async () => { + await stream.setKv("key", "value"); + expect(await stream.getKv("key")).toEqual("value"); + await wait({ until: () => stream2.getKv("key") == "value" }); + + // also use an empty '' key + const n = stream.length; + await stream.setKv("", "a value"); + await wait({ until: () => stream.length > n }); + expect(await stream.getKv("")).toEqual("a value"); + await wait({ until: () => stream2.getKv("") == "a value" }); + }); + + it("deletes the key and confirms it was deleted", async () => { + await stream.deleteKv("key"); + await wait({ until: () => stream.getKv("key") === undefined }); + await wait({ until: () => stream2.getKv("key") === undefined }); + }); + + it("also delete the empty key one", async () => { + await stream2.deleteKv(""); + await wait({ until: async () => (await stream2.getKv("")) == undefined }); + expect(await stream2.getKv("")).toEqual(undefined); + await wait({ until: () => stream.getKv("") === undefined }); + }); + + it("delete a key that doesn't exist -- a no-op (shouldn't make sequence longer)", async () => { + const n = stream.length; + await stream.deleteKv("fake"); + expect(stream.length).toBe(n); + }); + + it("cleans up", () => { + stream.close(); + stream2.close(); + client.close(); + client2.close(); + }); +}); + +describe("test previousSeq when setting keys, which can be used to ensure consistent read/writes", () => { + let client; + let stream; + let name = "prev"; + + it("creates persistent stream", async () => { + client = connect(); + stream = await cstream({ client, name, ephemeral: false }); + }); + + let seq; + it("sets a value", async () => { + const { seq: seq0 } = await stream.setKv("my", "value"); + expect(seq0).toBeGreaterThan(0); + seq = seq0; + }); + + it("tries to change the value using the wrong previousSeq", async () => { + await expect(async () => { + await stream.setKv("my", "newval", { previousSeq: 0 }); + }).rejects.toThrowError("wrong last sequence"); + }); + + it("changes the value using the correct previousSeq", async () => { + const { seq: seq1 } = await stream.setKv("my", "newval", { + previousSeq: seq, + }); + await wait({ until: () => stream.seqKv("my") == seq1 }); + expect(stream.getKv("my")).toBe("newval"); + expect(stream.seqKv("my")).toBe(seq1); + }); + + it("previousSeq is ignored with non-key sets", async () => { + const n = stream.length; + await stream.publish("stuff", { previousSeq: 0 }); + await wait({ until: () => stream.length > n }); + expect(stream.get(stream.length - 1)).toBe("stuff"); + }); +}); + +describe("test msgID dedup", () => { + let client; + let stream; + let name = "msgid"; + let client2; + let stream2; + + it("creates two clients", async () => { + client = connect(); + stream = await cstream({ client, name, ephemeral: false }); + + client2 = connect(); + stream2 = await cstream({ + client: client2, + name, + ephemeral: false, + noCache: true, + }); + + expect(stream === stream2).toBe(false); + }); + + it("publishes a message with msgID twice and sees it only appears once", async () => { + await stream.publish("x", { msgID: "myid" }); + await stream.publish("y", { msgID: "myid2" }); + await stream.publish("x", { msgID: "myid" }); + await wait({ until: () => stream.length == 2 }); + expect(stream.getAll()).toEqual(["x", "y"]); + await wait({ until: () => stream2.length == 2 }); + expect(stream2.getAll()).toEqual(["x", "y"]); + expect(stream.msgIDs.has("myid")).toBe(true); + }); + + it("publishes same message from other stream doesn't cause it to appear again either (so msgID check is server side)", async () => { + // not just using local info and not accidentally the same object: + expect(stream2.msgIDs.has("myid")).toBe(false); + await stream2.publish("x", { msgID: "myid" }); + expect(stream2.getAll()).toEqual(["x", "y"]); + await stream2.publish("y", { msgID: "myid2" }); + expect(stream2.getAll()).toEqual(["x", "y"]); + }); +}); + +import { disablePermissionCheck } from "@cocalc/conat/persist/client"; + +describe("test permissions", () => { + it("create a CoreStream, but change the path to one that wouldn't be allowed given the subject", async () => { + const client = connect(); + const stream: any = new CoreStream({ + client, + name: "conat.ipynb", + project_id: "00000000-0000-4000-8000-000000000000", + }); + expect(stream.storage.path).toBe( + "projects/00000000-0000-4000-8000-000000000000/conat.ipynb", + ); + expect(stream.user).toEqual({ + project_id: "00000000-0000-4000-8000-000000000000", + }); + + // now change it to something invalid by directly editing it + stream.storage.path = "hub/conat.ipynb"; + + // When we try to init, it must fail because the subject we use + // for our location (the 'user', defined by + // project_id: "00000000-0000-4000-8000-000000000000" + // above) doesn't give permissions to hub/. + // NOTE: even if a browser client is accessing a project resource + // they give the project_id, not their id. + await expect(async () => { + await stream.init(); + }).rejects.toThrowError("permission denied"); + + stream.close(); + }); + + it("do the tests again, but with the client side permission check disabled, to make sure the server denies us", async () => { + disablePermissionCheck(); + const client = connect(); + let stream: any = new CoreStream({ + client, + name: "conat2.ipynb", + project_id: "00000000-0000-4000-8000-000000000000", + }); + const origPath = stream.storage.path; + stream.storage.path = "hub/conat2.ipynb"; + await expect(async () => { + await stream.init(); + }).rejects.toThrowError("permission denied"); + stream.close(); + + stream = new CoreStream({ + client, + name: "conat2.ipynb", + project_id: "00000000-0000-4000-8000-000000000000", + }); + // instead change the user and make sure denied + stream.storage.path = origPath; + // wrong project + stream.user = { project_id: "00000000-0000-4000-8000-000000000004" }; + await expect(async () => { + await stream.init(); + }).rejects.toThrowError("permission denied"); + stream.close(); + + stream = new CoreStream({ + client, + name: "conat2.ipynb", + project_id: "00000000-0000-4000-8000-000000000000", + }); + stream.storage.path = origPath; + // wrong user type + stream.user = { account_id: "00000000-0000-4000-8000-000000000000" }; + await expect(async () => { + await stream.init(); + }).rejects.toThrowError("permission denied"); + + stream.close(); + }); +}); + +describe("test creating and closing a core-stream doesn't leak subscriptions", () => { + let client; + let stream; + let name = "sub.count"; + let subs; + + it("make a new client and count subscriptions", async () => { + client = connect(); + await once(client, "connected"); + await client.getInbox(); + subs = client.numSubscriptions(); + expect(subs).toBe(1); // the inbox + }); + + it("creates persistent stream", async () => { + stream = await cstream({ client, name, ephemeral: false }); + await stream.setKv("my", "value"); + expect(client.numSubscriptions()).toBe(2); + }); + + it("close the stream and confirm subs returns to 1", async () => { + stream.close(); + await expect(() => { + client.numSubscriptions() == 1; + }); + }); +}); + +afterAll(after); diff --git a/src/packages/backend/conat/test/core/services.test.ts b/src/packages/backend/conat/test/core/services.test.ts new file mode 100644 index 0000000000..749600ce89 --- /dev/null +++ b/src/packages/backend/conat/test/core/services.test.ts @@ -0,0 +1,189 @@ +/* +pnpm test ./services.test.ts +*/ + +import { + before, + after, + connect, + delay, +} from "@cocalc/backend/conat/test/setup"; +import { Client, type Message } from "@cocalc/conat/core/client"; +import { wait } from "@cocalc/backend/conat/test/util"; + +beforeAll(before); + +describe("test creating subscriptions", () => { + let client1, client2; + it("create two clients", async () => { + client1 = connect({ reconnectionDelay: 250 }); + client2 = connect(); + }); + + let sub; + it("create a subscription in client1 and make sure it can be used from client2", async () => { + sub = await client1.subscribe("foo"); + const { count } = await client2.publish("foo", "hello"); + expect(count).toBe(1); + const { value } = await sub.next(); + expect(value.data).toBe("hello"); + }); + + it("disconnects client1 and observes that client2 doesn't think client1 is listening anymore, rather than having requests 'hang until timeout'", async () => { + client1.conn.io.engine.close(); + await wait({ + until: async () => { + const { count } = await client2.publish("foo", "hello"); + return count == 0; + }, + }); + }); + + it("waits for client1 to connect again and observes that it *does* start receiving messages", async () => { + await wait({ + until: async () => { + const { count } = await client2.publish("foo", "hello"); + return count == 1; + }, + }); + }); + + it("cleans up", () => { + sub.close(); + client1.close(); + client2.close(); + }); +}); + +describe("more service tests", () => { + let client1: Client, client2: Client; + it("create two clients", async () => { + client1 = connect({ reconnectionDelay: 1000 }); + client2 = connect(); + }); + + let service, arith; + it("create a *service* with typing, subject in client1 and use it from client2", async () => { + interface Api { + add: (a: number, b: number) => Promise; + mul: (a: number, b: number) => Promise; + subject: (a: number, b: number) => Promise; + } + service = await client1.service("arith.*", { + add: async (a, b) => a + b, + mul: async (a, b) => a * b, + // Here we do NOT use an arrow => function and this is + // bound to the calling mesg, which lets us get the subject. + // Because user identity and permissions are done via wildcard + // subjects, having access to the calling message is critical + async subject(a, b) { + const mesg: Message = this as any; + return `${mesg.subject}-${a}-${b}`; + }, + }); + + arith = client2.call("arith.one"); + expect(await arith.mul(2, 3)).toBe(6); + expect(await arith.add(2, 3)).toBe(5); + + const arith2 = client2.call("arith.two"); + expect(await arith2.subject(2, 3)).toBe("arith.two-2-3"); + }); + + it("tests disconnect", async () => { + client1.conn.io.engine.close(); + await wait({ + until: async () => { + const { count } = await client2.publish("arith.one", "hello"); + return count == 0; + }, + }); + await expect(async () => { + await arith.mul(2, 3); + }).rejects.toThrowError("no subscribers"); + }); + + it("cleans up", () => { + service.close(); + client1.close(); + client2.close(); + }); +}); + +describe("illustrate using callMany to call multiple services and get all the results as an iterator", () => { + let client1: Client, client2: Client, client3: Client; + it("create three clients", async () => { + client1 = connect(); + client2 = connect(); + client3 = connect(); + }); + + let service1, service2; + interface Api { + who: () => Promise; + } + it("create simple service on client1 and client2", async () => { + service1 = await client1.service( + "whoami", + { + who: async () => { + return 1; + }, + }, + { queue: "1" }, + ); + service2 = await client2.service( + "whoami", + { + who: async () => { + // make this one slow: + await delay(250); + return 2; + }, + }, + { queue: "2" }, + ); + }); + + it("call it without callMany -- this actually sends the request to *both* servers and returns the quicker one.", async () => { + const call = client3.call("whoami"); + // quicker one is always 1: + expect(await call.who()).toBe(1); + expect(await call.who()).toBe(1); + expect(await call.who()).toBe(1); + }); + + it("call the service using callMany and get TWO results in parallel", async () => { + const callMany = client3.callMany("whoami", { maxWait: 500 }); + const X: number[] = []; + const start = Date.now(); + for await (const a of await callMany.who()) { + X.push(a); + } + expect(X.length).toBe(2); + expect(new Set(X)).toEqual(new Set([1, 2])); + expect(Date.now() - start).toBeGreaterThan(500); + }); + + it("call the service using callMany but limit results using mesgLimit instead of time", async () => { + const callMany = client3.callMany("whoami", { maxMessages: 2 }); + const X: number[] = []; + const start = Date.now(); + for await (const a of await callMany.who()) { + X.push(a); + } + expect(X.length).toBe(2); + expect(new Set(X)).toEqual(new Set([1, 2])); + expect(Date.now() - start).toBeLessThan(500); + }); + + it("cleans up", () => { + service1.close(); + service2.close(); + client1.close(); + client2.close(); + client3.close(); + }); +}); + +afterAll(after); diff --git a/src/packages/backend/nats/test/files/read.test.ts b/src/packages/backend/conat/test/files/read.test.ts similarity index 92% rename from src/packages/backend/nats/test/files/read.test.ts rename to src/packages/backend/conat/test/files/read.test.ts index 9a72326516..ef87e46139 100644 --- a/src/packages/backend/nats/test/files/read.test.ts +++ b/src/packages/backend/conat/test/files/read.test.ts @@ -4,12 +4,15 @@ Test async streaming read of files from a compute servers using NATS. DEVELOPMENT: -pnpm exec jest --watch --forceExit --detectOpenHandles "read.test.ts" +pnpm test ./read.test.ts */ -import "@cocalc/backend/nats"; -import { close, createServer, readFile } from "@cocalc/nats/files/read"; +import { before, after } from "@cocalc/backend/conat/test/setup"; + +beforeAll(before); + +import { close, createServer, readFile } from "@cocalc/conat/files/read"; import { createReadStream } from "fs"; import { file as tempFile } from "tmp-promise"; import { writeFile as fsWriteFile } from "fs/promises"; @@ -103,3 +106,6 @@ describe("do a larger test that involves multiple chunks and a different name", } }); }); + + +afterAll(after); diff --git a/src/packages/backend/nats/test/files/write.test.ts b/src/packages/backend/conat/test/files/write.test.ts similarity index 85% rename from src/packages/backend/nats/test/files/write.test.ts rename to src/packages/backend/conat/test/files/write.test.ts index 9ce062f95e..b7c7644eef 100644 --- a/src/packages/backend/nats/test/files/write.test.ts +++ b/src/packages/backend/conat/test/files/write.test.ts @@ -4,12 +4,15 @@ Test async streaming writing of files to compute servers using NATS. DEVELOPMENT: - pnpm exec jest --watch --forceExit --detectOpenHandles "write.test.ts" + pnpm test ./write.test.ts */ -import "@cocalc/backend/nats"; -import { close, createServer, writeFile } from "@cocalc/nats/files/write"; +import { before, after } from "@cocalc/backend/conat/test/setup"; + +beforeAll(before); + +import { close, createServer, writeFile } from "@cocalc/conat/files/write"; import { createWriteStream, createReadStream } from "fs"; import { file as tempFile } from "tmp-promise"; import { writeFile as fsWriteFile, readFile } from "fs/promises"; @@ -111,7 +114,13 @@ describe("do a more challenging test that involves a larger file that has to be }); it("confirm that the dest file is correct", async () => { - const d = (await readFile(dest)).toString(); + let d = (await readFile(dest)).toString(); + if (d.length != CONTENT.length) { + // under heavy load file might not have been flushed **to disk** (even though it was fully and + // correctly received), so we wait to give it a chance, then try again. + await delay(1000); + d = (await readFile(dest)).toString(); + } expect(d.length).toEqual(CONTENT.length); // not directly comparing, since huge and if something goes wrong the output // saying the test failed is huge. @@ -125,3 +134,5 @@ describe("do a more challenging test that involves a larger file that has to be } }); }); + +afterAll(after); diff --git a/src/packages/backend/nats/test/llm.test.ts b/src/packages/backend/conat/test/llm.test.ts similarity index 86% rename from src/packages/backend/nats/test/llm.test.ts rename to src/packages/backend/conat/test/llm.test.ts index 74929fae43..9ac69af07f 100644 --- a/src/packages/backend/nats/test/llm.test.ts +++ b/src/packages/backend/conat/test/llm.test.ts @@ -3,15 +3,18 @@ Test LLM NATS streaming. DEVELOPMENT: - pnpm exec jest --watch --forceExit --detectOpenHandles "llm.test.ts" + pnpm test llm.test.ts */ // this sets client -import "@cocalc/backend/nats"; +import "@cocalc/backend/conat"; -import { init, close } from "@cocalc/nats/llm/server"; -import { llm } from "@cocalc/nats/llm/client"; +import { init, close } from "@cocalc/conat/llm/server"; +import { llm } from "@cocalc/conat/llm/client"; +import { before, after } from "@cocalc/backend/conat/test/setup"; + +beforeAll(before); describe("create an llm server, client, and stub evaluator, and run an evaluation", () => { // define trivial evaluate @@ -82,3 +85,5 @@ describe("test an evaluate that throws an error half way through", () => { await close(); }); }); + +afterAll(after); diff --git a/src/packages/backend/conat/test/persist/README.md b/src/packages/backend/conat/test/persist/README.md new file mode 100644 index 0000000000..09299a61f0 --- /dev/null +++ b/src/packages/backend/conat/test/persist/README.md @@ -0,0 +1,2 @@ +Most tests of persist are in other test directories, via creating streams, +etc. However, here we directly test some things. diff --git a/src/packages/backend/conat/test/persist/multiple-servers.test.ts b/src/packages/backend/conat/test/persist/multiple-servers.test.ts new file mode 100644 index 0000000000..dd35803217 --- /dev/null +++ b/src/packages/backend/conat/test/persist/multiple-servers.test.ts @@ -0,0 +1,121 @@ +/* +Unit tests of multiple persist servers at once: + +- making numerous distinct clients and seeing that they are distributed across persist servers +- stopping a persist server and seeing that failover happens without data loss + +pnpm test `pwd`/multiple-servers.test.ts + +*/ + +import { + before, + after, + connect, + persistServer as defaultPersistServer, + once, +} from "@cocalc/backend/conat/test/setup"; +import { stream } from "@cocalc/conat/persist/client"; +import { delay } from "awaiting"; +import { server as createPersistServer } from "@cocalc/backend/conat/persist"; +import { messageData } from "@cocalc/conat/core/client"; +import { uuid } from "@cocalc/util/misc"; + +beforeAll(before); + +describe("multiple clients using multiple persist servers", () => { + const persistServers: any[] = []; + let numServers = 4; + it(`ceate ${numServers} persist servers`, async () => { + persistServers.push(defaultPersistServer); + for (let i = 0; i < numServers - 1; i++) { + const client = connect(); + const persistServer = createPersistServer({ client }); + await once(persistServer, "ready"); + persistServers.push(persistServer); + } + }); + + let persistClients: any[] = []; + + // NOTE: count must be below about 40 to avoid hitting the default + // per-user connection limit of 100. + let count = 30; + const projects: string[] = []; + it(`creates ${count} persist clients`, async () => { + const ids = new Set([]); + for (let i = 0; i < count; i++) { + const client = connect(); + const project_id = uuid(); + projects.push(project_id); + const persistClient = stream({ + client, + user: { project_id }, + storage: { path: `projects/${project_id}/foo-${i}` }, + }); + ids.add(await persistClient.serverId()); + persistClients.push(persistClient); + const { seq } = await persistClient.set({ + messageData: messageData(i, { headers: { [i]: i } }), + }); + expect(seq).toBe(1); + } + // given that we're randomly distributing so many clients (what really matters + // is the user field above) + // it's highly likely we hit all servers. + expect(ids.size).toBe(persistServers.length); + }); + + it(`add ${numServers} more persist servers`, async () => { + for (let i = 0; i < numServers - 1; i++) { + const client = connect(); + const persistServer = createPersistServer({ client }); + await once(persistServer, "ready"); + persistServers.push(persistServer); + } + }); + + it("read data we wrote above (so having new servers doesn't mess with existing connections)", async () => { + for (let i = 0; i < count; i++) { + const mesg = await persistClients[i].get({ seq: 1 }); + expect(mesg.data).toBe(i); + expect(mesg.headers[`${i}`]).toBe(i); + } + }); + + it("new clients use exactly the same servers, since the assignment was already made above", async () => { + const ids = new Set([]); + for (let i = 0; i < count; i++) { + const client = connect(); + const project_id = projects[i]; + const persistClient = stream({ + client, + user: { project_id }, + storage: { path: `projects/${project_id}/foo-${i}` }, + }); + ids.add(await persistClient.serverId()); + persistClients.push(persistClient); + const { seq } = await persistClient.set({ + messageData: messageData(i, { headers: { [i]: i } }), + }); + expect(seq).toBe(2); + } + expect(ids.size).toBe(numServers); + }); + + it("cleans up", () => { + for (const client of persistClients) { + client.close(); + } + for (const server of persistServers) { + server.close(); + } + }); +}); + +afterAll(async () => { + // slight delay so all the sqlites of the persist servers can finish + // writing to disk, so we can delete the temp directory + await delay(100); + await after(); +}); diff --git a/src/packages/backend/conat/test/persist/permissions.test.ts b/src/packages/backend/conat/test/persist/permissions.test.ts new file mode 100644 index 0000000000..ed55d7da6b --- /dev/null +++ b/src/packages/backend/conat/test/persist/permissions.test.ts @@ -0,0 +1,90 @@ +/* + +pnpm test ./permissions.test.ts + +*/ +import { SERVICE } from "@cocalc/conat/persist/util"; +import { assertHasWritePermission } from "@cocalc/conat/persist/auth"; + +const uuid = "00000000-0000-4000-8000-000000000000"; +const uuid2 = "00000000-0000-4000-8000-000000000002"; + +describe("test subject permissions directly by calling assertHasWritePermission", () => { + it("checks a bunch of things that should work don't throw", () => { + // these don't throw + assertHasWritePermission({ + subject: `${SERVICE}.hub`, + path: "hub/foo", + }); + + assertHasWritePermission({ + subject: `${SERVICE}.hub`, + path: "hub/foo/blah xxx~!/xxxx", + }); + + assertHasWritePermission({ + subject: `${SERVICE}.project-${uuid}`, + path: `projects/${uuid}/a.txt`, + }); + + assertHasWritePermission({ + subject: `${SERVICE}.account-${uuid}`, + path: `accounts/${uuid}/c/d.txt`, + }); + }); + + it("now check many things that are NOT allowed", () => { + const BAD = [ + { subject: `${SERVICE}.fubar`, path: "hub/foo/bar" }, + { subject: `fluber.hub`, path: "hub/foo" }, + { + subject: `${SERVICE}.projects-${uuid}`, + path: `projects/${uuid}/foo`, + }, + { + subject: `${SERVICE}.accounts-${uuid}`, + path: `accounts/${uuid}/foo`, + }, + { + subject: `${SERVICE}.project-${uuid}`, + path: `accounts/${uuid}/foo`, + }, + { + subject: `${SERVICE}.account-${uuid}`, + path: `projects/${uuid}/foo`, + }, + { + subject: `${SERVICE}.account-${uuid}`, + path: `accounts/${uuid2}/foo`, + }, + { + subject: `${SERVICE}.project-${uuid}`, + path: `projects/${uuid2}/foo`, + }, + { + subject: `${SERVICE}.project-${uuid}`, + path: `projects/${uuid}/`, + }, + { + subject: `${SERVICE}.project-${uuid}`, + path: `projects/${uuid}`, + }, + { + subject: `${SERVICE}.project-${uuid}`, + path: `projects/${uuid}/foo/`, + }, + { + subject: `${SERVICE}.project-${uuid}`, + path: `projects/${uuid}/${"a".repeat(100000)}`, + }, + { + subject: `${SERVICE}.project-${uuid}x`, + path: `projects/${uuid}x/a.txt`, + }, + ]; + + for (const { subject, path } of BAD) { + expect(() => assertHasWritePermission({ subject, path })).toThrow(); + } + }); +}); diff --git a/src/packages/backend/conat/test/persist/persist-client-restarts.test.ts b/src/packages/backend/conat/test/persist/persist-client-restarts.test.ts new file mode 100644 index 0000000000..e89d687ef3 --- /dev/null +++ b/src/packages/backend/conat/test/persist/persist-client-restarts.test.ts @@ -0,0 +1,111 @@ +/* +Tests of persist client. + +pnpm test ./persist-client.test.ts + +*/ + +import { + before, + after, + connect, + restartServer, + restartPersistServer, + wait, +} from "@cocalc/backend/conat/test/setup"; +import { stream } from "@cocalc/conat/persist/client"; +import { messageData } from "@cocalc/conat/core/client"; + +beforeAll(before); + +describe("restarting the network and/or persist server, but with no delay afterwards", () => { + let client, s1; + + it("creates a client, stream and test data", async () => { + client = connect(); + s1 = stream({ + client, + user: { hub_id: "x" }, + storage: { path: "hub/network" }, + }); + await s1.set({ + key: "test", + messageData: messageData("data"), + }); + const mesg = await s1.get({ key: "test" }); + expect(mesg.data).toBe("data"); + }); + + it("restart conat networking", async () => { + await restartServer(); + }); + + it("it start working again after restart of socketio server only, though we expect some errors", async () => { + try { + await s1.get({ key: "test", timeout: 500 }); + } catch (err) { + expect(`${err}`).toMatch(/timeout|subscribers|disconnected/); + } + await wait({ + until: async () => { + try { + await s1.get({ key: "test", timeout: 500 }); + return true; + } catch { + return false; + } + }, + }); + const mesg = await s1.get({ key: "test" }); + expect(mesg.data).toBe("data"); + }); + + it("restarts just persist server", () => { + restartPersistServer(); + }); + + it("it starts working again after restart after persist server only, though we expect some errors", async () => { + await wait({ + until: async () => { + try { + await s1.set({ + key: "test-5", + messageData: messageData("data", { headers: { foo: "bar" } }), + timeout: 500, + }); + return true; + } catch (err) { + return false; + } + }, + }); + const mesg = await s1.get({ key: "test-5" }); + expect(mesg.data).toBe("data"); + }); + + it("restarts BOTH the socketio server and the persist server", () => { + restartServer(); + restartPersistServer(); + }); + + it("it starts working again after restart of BOTH servers, though we expect some errors", async () => { + await wait({ + until: async () => { + try { + await s1.set({ + key: "test-10", + messageData: messageData("data", { headers: { foo: "bar" } }), + timeout: 500, + }); + return true; + } catch (err) { + return false; + } + }, + }); + const mesg = await s1.get({ key: "test-10" }); + expect(mesg.data).toBe("data"); + }); +}); + +afterAll(after); diff --git a/src/packages/backend/conat/test/persist/persist-client.test.ts b/src/packages/backend/conat/test/persist/persist-client.test.ts new file mode 100644 index 0000000000..0381bc72f4 --- /dev/null +++ b/src/packages/backend/conat/test/persist/persist-client.test.ts @@ -0,0 +1,305 @@ +/* +Tests of persist client. + +pnpm test ./persist-client.test.ts + +*/ + +import { + before, + after, + connect, + restartServer, + restartPersistServer, + wait, +} from "@cocalc/backend/conat/test/setup"; +import { stream } from "@cocalc/conat/persist/client"; +import { messageData } from "@cocalc/conat/core/client"; +import { delay } from "awaiting"; + +beforeAll(before); + +jest.setTimeout(10000); +describe("create a persist client stream and test the basic operations", () => { + let client, s1; + + it("creates a client and stream", () => { + client = connect(); + s1 = stream({ + client, + user: { hub_id: "x" }, + storage: { path: "hub/foo" }, + }); + }); + + let seq0; + it("write a value to the stream", async () => { + const { seq, time } = await s1.set({ + messageData: messageData("hi", { headers: { foo: "bar" } }), + }); + expect(Math.abs(time - Date.now())).toBeLessThan(1000); + seq0 = seq; + }); + + it("get the value back", async () => { + const mesg = await s1.get({ seq: seq0 }); + expect(mesg.data).toBe("hi"); + expect(mesg.headers.foo).toBe("bar"); + }); + + it("writes a value with a key", async () => { + await s1.set({ + key: "my-key", + messageData: messageData("value", { headers: { foo: "bar" } }), + }); + const mesg = await s1.get({ key: "my-key" }); + expect(mesg.data).toBe("value"); + }); +}); + +describe("restarting persist server", () => { + let client, s1; + + it("creates a client and stream and write test data", async () => { + client = connect(); + s1 = stream({ + client, + user: { hub_id: "x" }, + storage: { path: "hub/bar" }, + }); + await s1.set({ + key: "test", + messageData: messageData("data", { headers: { foo: "bar" } }), + }); + }); + + it("restart the persist server", async () => { + await restartPersistServer(); + }); + + it("first attempt to read the data written above fails because persist server hasn't started yet", async () => { + await expect(async () => { + await s1.get({ key: "test", timeout: 500 }); + }).rejects.toThrow("no subscribers"); + }); + + it("it does start working relatively quickly though", async () => { + await wait({ + until: async () => { + try { + await s1.get({ key: "test", timeout: 1500 }); + return true; + } catch {} + }, + }); + + const mesg = await s1.get({ key: "test" }); + expect(mesg.data).toBe("data"); + }); +}); + +describe("restarting persist server with an ephemeral stream", () => { + let client, s1; + + it("creates a client and an ephemeral stream and write test data", async () => { + client = connect(); + s1 = stream({ + client, + user: { hub_id: "x" }, + storage: { path: "hub/in-memory-only", ephemeral: true }, + }); + await s1.set({ + key: "test", + messageData: messageData("data", { headers: { foo: "bar" } }), + }); + }); + + it("restart the persist server", async () => { + await restartPersistServer(); + }); + + it("our data is gone - it's ephemeral", async () => { + s1 = stream({ + client, + user: { hub_id: "x" }, + storage: { path: "hub/in-memory-onl", ephemeral: true }, + }); + await wait({ + until: async () => { + try { + const mesg = await s1.get({ key: "test", timeout: 500 }); + return mesg === undefined; + } catch {} + }, + }); + + expect(await s1.get({ key: "test" })).toBe(undefined); + }); +}); + +describe("restarting the network but not the persist server", () => { + let client, s1; + + it("creates a client and stream and write test data", async () => { + client = connect(); + s1 = stream({ + client, + user: { hub_id: "x" }, + storage: { path: "hub/network" }, + }); + await s1.set({ + key: "test", + messageData: messageData("data", { headers: { foo: "bar" } }), + }); + }); + + it("restart conat networking", async () => { + await restartServer(); + }); + + it("it does start working eventually", async () => { + await wait({ + until: async () => { + try { + await s1.get({ key: "test", timeout: 1000 }); + return true; + } catch {} + }, + }); + const mesg = await s1.get({ key: "test" }); + expect(mesg.data).toBe("data"); + }); +}); + +describe("test a changefeed", () => { + let client, s1, cf; + + it("creates a client, stream and changefeed", async () => { + client = connect(); + s1 = stream({ + client, + user: { hub_id: "x" }, + storage: { path: "hub/changefeed" }, + }); + cf = await s1.changefeed(); + }); + + it("write and see result via changefeed", async () => { + await s1.set({ + key: "test", + messageData: messageData("data", { headers: { foo: "bar" } }), + }); + const { value, done } = await cf.next(); + expect(done).toBe(false); + expect(value.seq).toBe(0); + expect(value.updates[0]).toEqual( + expect.objectContaining({ + op: "set", + seq: 1, + key: "test", + headers: { foo: "bar" }, + }), + ); + }); + + let s2, client2; + it("write via another client and see result via changefeed", async () => { + client2 = connect(); + s2 = stream({ + client: client2, + user: { hub_id: "x" }, + storage: { path: "hub/changefeed" }, + }); + expect(s1).not.toBe(s2); + await s2.set({ + key: "test2", + messageData: messageData("data2", { headers: { foo: "bar2" } }), + }); + + const { value, done } = await cf.next(); + expect(done).toBe(false); + expect(value.seq).toBe(1); + expect(value.updates[0]).toEqual( + expect.objectContaining({ + op: "set", + seq: 2, + key: "test2", + headers: { foo: "bar2" }, + }), + ); + }); + + // this takes a while due to it having to deal with the network restart + it("restart conat socketio server, and verify changefeed still works", async () => { + await restartServer(); + await wait({ + until: async () => { + // this set is expected to fail while networking is restarting + try { + await s1.set({ + key: "test3", + messageData: messageData("data3", { headers: { foo: "bar3" } }), + timeout: 1000, + }); + return true; + } catch { + return false; + } + }, + start: 500, + }); + + // the changefeed should still work and detect the above write that succeeded. + const { value, done } = await cf.next(); + expect(done).toBe(false); + expect(value.updates[0]).toEqual( + expect.objectContaining({ + op: "set", + seq: 3, + key: "test3", + headers: { foo: "bar3" }, + }), + ); + }); + + it("restart the persist server -- this is pretty brutal", async () => { + await restartPersistServer(); + }); + + it("set still works (with error) after restarting persist server", async () => { + // doing this set should fail due to persist for a second due server being + // off and having to connect again. + await wait({ + until: async () => { + try { + await s2.set({ + key: "test4", + messageData: messageData("data4", { headers: { foo: "bar4" } }), + timeout: 500, + }); + + return true; + } catch { + return false; + } + }, + }); + const mesg = await s2.get({ key: "test4" }); + expect(mesg.data).toBe("data4"); + }); + + it("changefeed still works after restarting persist server, though what gets received is somewhat random -- the persist server doesn't have its own state so can't guarantee continguous changefeeds when it restarts", async () => { + await delay(1000); + await s2.set({ + key: "test5", + messageData: messageData("data5", { headers: { foo: "bar5" } }), + timeout: 1000, + }); + const { value, done } = await cf.next(); + expect(done).toBe(false); + // changefeed may or may not have dropped a message, depending on timing + expect(value.updates[0].headers?.foo?.startsWith("bar")).toBe(true); + }); +}); + +afterAll(after); diff --git a/src/packages/backend/conat/test/persist/persist-dstream.test.ts b/src/packages/backend/conat/test/persist/persist-dstream.test.ts new file mode 100644 index 0000000000..cdb494d429 --- /dev/null +++ b/src/packages/backend/conat/test/persist/persist-dstream.test.ts @@ -0,0 +1,50 @@ +/* +Testing things about dstream that involve persistence. +*/ + +import { + before, + after, + connect, + delay, +} from "@cocalc/backend/conat/test/setup"; + +beforeAll(before); + +describe("create a dstream, write data, close it, then open it again and see data persisted", () => { + let client; + const name = "foo"; + it("create clients and a document", async () => { + client = connect(); + const v = await client.sync.dstream({ name }); + v.publish("x"); + await v.save(); + v.close(); + }); + + it("opening again and see that it persisted", async () => { + const v = await client.sync.dstream({ name }); + expect(v.getAll()).toEqual(["x"]); + }); +}); + +// this is here because I had a bug in the syncRefCache that this exposed. +describe.only("just like above, create a dstream, write data, close it, then open it again and see data persisted -- **but do it all in the same function**", () => { + let client; + const name = "foo2"; + it("create clients and a document", async () => { + client = connect(); + const v = await client.sync.dstream({ name }); + v.publish("x"); + await v.save(); + v.close(); + const w = await client.sync.dstream({ name }); + expect(w.getAll()).toEqual(["x"]); + w.close(); + }); +}); + +afterAll(async () => { + await delay(100); + after(); +}); diff --git a/src/packages/backend/conat/test/server/limits.test.ts b/src/packages/backend/conat/test/server/limits.test.ts new file mode 100644 index 0000000000..5ea66d6a81 --- /dev/null +++ b/src/packages/backend/conat/test/server/limits.test.ts @@ -0,0 +1,34 @@ +/* +Test various configurable limits of the server. + +pnpm test ./limits.test.ts +*/ + +import { createServer } from "@cocalc/backend/conat/test/setup"; + +describe("test the per user subscription limit", () => { + let server; + + it("creates a server with a subscription limit of 3", async () => { + server = await createServer({ maxSubscriptionsPerClient: 3 }); + }); + + let client; + it("creates a client and makes 2 subscriptions fine", async () => { + // can't make a third, since the default INBOX subscription already counts. + client = server.client(); + await client.sub("sub1"); + await client.sub("sub2"); + }); + + it("creates another subscription and gets an error", async () => { + await expect(async () => { + await client.sub("sub3"); + }).rejects.toThrowError("limit"); + }); + + it("cleans up", () => { + client.close(); + server.close(); + }); +}); diff --git a/src/packages/backend/conat/test/server/pubsub.test.ts b/src/packages/backend/conat/test/server/pubsub.test.ts new file mode 100644 index 0000000000..2f9ec93ce6 --- /dev/null +++ b/src/packages/backend/conat/test/server/pubsub.test.ts @@ -0,0 +1,208 @@ +/* +Stress test pub/sub in various settings: + +- single server +- multiple servers connected via valkey +- multiple servers connected via valkey with a sticky subscription +- multiple servers connected via valkey with 800 existing subscriptions + +The goal is just to make sure there are no ridiculous performance regressions. +Also, this is a useful file to play around with to understand the speed +we should expect sending simple messages. It's basically + +- sending sync is about 30K/second + - receiving and processing on the client is about 3K/second + +The speed doesn't depend much on the situation above, though it goes down to +about 2K/second with valkey. It seems like we will need at least one socket.io +server per say 100 clients. + + +t ./pubsub-stress.test.ts + +*/ + +import { + before, + after, + initConatServer, + runValkey, +} from "@cocalc/backend/conat/test/setup"; +import { STICKY_QUEUE_GROUP } from "@cocalc/conat/core/client"; +import { waitForSubscription } from "./util"; + +// should be several thousand, so 250 seems reasonable as a cutoff to indicate +// things are horribly wrong +const REQUIRED_SINGLE_SERVER_RECV_MESSAGES_PER_SECOND = 250; +// should be tens of thousands +const REQUIRED_SINGLE_SERVER_SEND_MESSAGES_PER_SECOND = 500; + +const REQUIRED_VALKEY_SERVER_RECV_MESSAGES_PER_SECOND = 200; +const REQUIRED_VALKEY_SERVER_SEND_MESSAGES_PER_SECOND = 400; + +const VERBOSE = false; +const log = VERBOSE ? console.log : (..._args) => {}; + +beforeAll(before); + +jest.setTimeout(15000); + +describe("create two servers connected via valkey and two clients and test messaging speed", () => { + let server, client1, client2; + it("one server and two clients connected to it", async () => { + server = await initConatServer(); + client1 = server.client(); + client2 = server.client(); + }); + + const count1 = 1000; + it(`do a benchmark without valkey of send/receiving ${count1} messages`, async () => { + const sub = await client1.subscribe("bench"); + await waitForSubscription(server, "bench"); + const f = async () => { + const start = Date.now(); + let i = 0; + for await (const _ of sub) { + i += 1; + if (i >= count1) { + return Math.ceil((count1 / (Date.now() - start)) * 1000); + } + } + }; + const start = Date.now(); + for (let i = 0; i < count1; i++) { + client2.publishSync("bench", null); + } + const sendRate = Math.ceil((count1 / (Date.now() - start)) * 1000); + log("sent", sendRate, "messages per second"); + expect(sendRate).toBeGreaterThan( + REQUIRED_SINGLE_SERVER_SEND_MESSAGES_PER_SECOND, + ); + const recvRate = await f(); + log("received ", recvRate, "messages per second"); + expect(recvRate).toBeGreaterThan( + REQUIRED_SINGLE_SERVER_RECV_MESSAGES_PER_SECOND, + ); + }); + + const count2 = 1000; + it(`do a benchmark with valkey of send/receiving ${count2} messages`, async () => { + const valkeyServer = await runValkey(); + const valkey = valkeyServer.address; + const server1 = await initConatServer({ valkey }); + const server2 = await initConatServer({ valkey }); + const client1 = server1.client(); + const client2 = server2.client(); + const sub = await client1.subscribe("bench"); + await waitForSubscription(server1, "bench"); + await waitForSubscription(server2, "bench"); + const f = async () => { + const start = Date.now(); + let i = 0; + for await (const _ of sub) { + i += 1; + if (i >= count1) { + return Math.ceil((count1 / (Date.now() - start)) * 1000); + } + } + }; + const start = Date.now(); + for (let i = 0; i < count1; i++) { + client2.publishSync("bench", null); + } + const sendRate = Math.ceil((count1 / (Date.now() - start)) * 1000); + log("valkey: sent", sendRate, "messages per second"); + expect(sendRate).toBeGreaterThan( + REQUIRED_VALKEY_SERVER_SEND_MESSAGES_PER_SECOND, + ); + const recvRate = await f(); + log("valkey: received ", recvRate, "messages per second"); + expect(recvRate).toBeGreaterThan( + REQUIRED_VALKEY_SERVER_RECV_MESSAGES_PER_SECOND, + ); + }); + + it(`do a benchmark with valkey of send/receiving and STICKY SUB and ${count2} messages`, async () => { + const valkeyServer = await runValkey(); + const valkey = valkeyServer.address; + const server1 = await initConatServer({ valkey }); + const server2 = await initConatServer({ valkey }); + const client1 = server1.client(); + const client2 = server2.client(); + const sub = await client1.subscribe("bench", { queue: STICKY_QUEUE_GROUP }); + await waitForSubscription(server1, "bench"); + await waitForSubscription(server2, "bench"); + const f = async () => { + const start = Date.now(); + let i = 0; + for await (const _ of sub) { + i += 1; + if (i >= count1) { + return Math.ceil((count1 / (Date.now() - start)) * 1000); + } + } + }; + const start = Date.now(); + for (let i = 0; i < count1; i++) { + client2.publishSync("bench", null); + } + const sendRate = Math.ceil((count1 / (Date.now() - start)) * 1000); + log("sticky valkey: sent", sendRate, "messages per second"); + expect(sendRate).toBeGreaterThan( + REQUIRED_VALKEY_SERVER_SEND_MESSAGES_PER_SECOND, + ); + const recvRate = await f(); + log("sticky valkey: received ", recvRate, "messages per second"); + expect(recvRate).toBeGreaterThan( + REQUIRED_VALKEY_SERVER_RECV_MESSAGES_PER_SECOND, + ); + }); + + const subcount = 400; + it(`do a benchmark with valkey of send/receiving and ${count2} messages after adding ${subcount} random subscriptions per client`, async () => { + const valkeyServer = await runValkey(); + const valkey = valkeyServer.address; + const server1 = await initConatServer({ valkey }); + const server2 = await initConatServer({ valkey }); + const client1 = server1.client(); + const client2 = server2.client(); + + const v: any[] = []; + for (let i = 0; i < subcount; i++) { + v.push(await client1.subscribe(`bench.one.${i}`)); + v.push(await client2.subscribe(`bench.two.${i}`)); + } + await waitForSubscription(server1, `bench.one.${subcount - 1}`); + await waitForSubscription(server2, `bench.two.${subcount - 1}`); + + const sub = await client1.subscribe("bench", { queue: STICKY_QUEUE_GROUP }); + await waitForSubscription(server1, "bench"); + await waitForSubscription(server2, "bench"); + const f = async () => { + const start = Date.now(); + let i = 0; + for await (const _ of sub) { + i += 1; + if (i >= count1) { + return Math.ceil((count1 / (Date.now() - start)) * 1000); + } + } + }; + const start = Date.now(); + for (let i = 0; i < count1; i++) { + client2.publishSync("bench", null); + } + const sendRate = Math.ceil((count1 / (Date.now() - start)) * 1000); + log("many subs + valkey: sent", sendRate, "messages per second"); + expect(sendRate).toBeGreaterThan( + REQUIRED_VALKEY_SERVER_SEND_MESSAGES_PER_SECOND, + ); + const recvRate = await f(); + log("many subs + valkey: received ", recvRate, "messages per second"); + expect(recvRate).toBeGreaterThan( + REQUIRED_VALKEY_SERVER_RECV_MESSAGES_PER_SECOND, + ); + }); +}); + +afterAll(after); diff --git a/src/packages/backend/conat/test/server/util.ts b/src/packages/backend/conat/test/server/util.ts new file mode 100644 index 0000000000..110d69dfda --- /dev/null +++ b/src/packages/backend/conat/test/server/util.ts @@ -0,0 +1,25 @@ +import { wait } from "@cocalc/backend/conat/test/setup"; + +export async function waitForSubscription(server, subject) { + await wait({ + until: () => { + return server.interest.patterns[subject] !== undefined; + }, + }); +} + +export async function waitForNonSubscription(server, subject) { + await wait({ + until: () => { + return server.interest.patterns[subject] === undefined; + }, + }); +} + +export async function waitForSticky(server, subject) { + await wait({ + until: () => { + return server.sticky[subject] !== undefined; + }, + }); +} diff --git a/src/packages/backend/conat/test/server/valkey-trim-streams.test.ts b/src/packages/backend/conat/test/server/valkey-trim-streams.test.ts new file mode 100644 index 0000000000..43bb756614 --- /dev/null +++ b/src/packages/backend/conat/test/server/valkey-trim-streams.test.ts @@ -0,0 +1,48 @@ +/* +Test automatically trimming the valkey streams used to coordinate subscription +interest and sticky subject choices. + +pnpm test ./valkey-trim-streams.test.ts +*/ + +import { + before, + after, + initConatServer, + runValkey, +} from "@cocalc/backend/conat/test/setup"; +import { waitForSubscription } from "./util"; + +beforeAll(before); + +// this is very important, since the sticky resolution needs to be consistent +describe("create two servers connected via valkey and observe stream trimming", () => { + let server1, server2, client1, client2, valkeyServer; + // we configure very aggressive trimming -- every 500ms we delete everything older than 1 seconds. + const opts = { valkeyTrimMaxAge: 1000, valkeyTrimInterval: 500 }; + it("create servers and clients", async () => { + valkeyServer = await runValkey(); + const valkey = valkeyServer.address; + server1 = await initConatServer({ + valkey, + ...opts, + }); + server2 = await initConatServer({ + valkey, + ...opts, + }); + client1 = server1.client(); + client2 = server2.client(); + }); + + it("test that the stream is working at all", async () => { + const subject = "stream.trim"; + const sub = await client1.subscribe(subject); + await waitForSubscription(server1, subject); + await waitForSubscription(server2, subject); + client2.publish(subject, "foo"); + await sub.next(); + }); +}); + +afterAll(after); diff --git a/src/packages/backend/conat/test/server/valkey.test.ts b/src/packages/backend/conat/test/server/valkey.test.ts new file mode 100644 index 0000000000..80c7b622cf --- /dev/null +++ b/src/packages/backend/conat/test/server/valkey.test.ts @@ -0,0 +1,252 @@ +/* +Test using socket servers connected via valkey. + +pnpm test ./valkey.test.ts +*/ + +import { + before, + after, + client, + server, + initConatServer, + delay, + runValkey, + wait, +} from "@cocalc/backend/conat/test/setup"; +import { STICKY_QUEUE_GROUP } from "@cocalc/conat/core/client"; +import { + waitForSubscription, + waitForNonSubscription, + waitForSticky, +} from "./util"; + +beforeAll(before); + +describe("create two conat socket servers NOT connected via a valkey stream, and observe communication is totally broken (of course)", () => { + let server2; + it("creates a second server", async () => { + server2 = await initConatServer(); + expect(server.options.port).not.toEqual(server2.options.port); + }); + + let client2; + it("observe client connected to each server CAN'T communicate", async () => { + client2 = server2.client(); + const sub = await client.subscribe("subject"); + // this will never be seen by client: + client2.publish("subject", "from client 2"); + await delay(500); + // this does get seen by client + client.publish("subject", "from client 1"); + const { value } = await sub.next(); + expect(value.data).toBe("from client 1"); + }); + + it("cleans up", () => { + client2.close(); + server2.close(); + }); +}); + +describe("do the same setup as above with two servers, but connected via valkey, and see that they do communicate", () => { + let server1, server2, valkey, valkeyServer; + it("creates valkey and two servers", async () => { + valkeyServer = await runValkey(); + valkey = valkeyServer.address; + server1 = await initConatServer({ valkey }); + + // configuration for valkey can also be given as a json string: + server2 = await initConatServer({ + valkey: JSON.stringify({ + password: valkeyServer.password, + port: valkeyServer.port, + }), + }); + expect(server1.options.port).not.toEqual(server2.options.port); + }); + + let client1; + let client2; + let sub1; + const SUBJECT = "my-subject.org"; + const SUBJECT2 = "my-subject2.org"; + it("create client connected to each server and verify that they CAN communicate with each other via pub/sub", async () => { + client1 = server1.client(); + client2 = server2.client(); + + sub1 = await client1.subscribe(SUBJECT); + await waitForSubscription(server2, SUBJECT); + + expect(Object.keys(server1.interest.patterns)).toContain(SUBJECT); + expect(Object.keys(server2.interest.patterns)).toContain(SUBJECT); + client2.publish(SUBJECT, "from client 2"); + const { value } = await sub1.next(); + expect(value.data).toBe("from client 2"); + + const sub2 = await client2.subscribe(SUBJECT2); + await waitForSubscription(server1, SUBJECT2); + + client1.publish(SUBJECT2, "hi from client 1"); + const { value: value2 } = await sub2.next(); + expect(value2.data).toBe("hi from client 1"); + }); + + it("client unsubscribes and that is reflected in both servers", async () => { + sub1.close(); + await waitForNonSubscription(server1, SUBJECT); + await waitForNonSubscription(server2, SUBJECT); + expect(Object.keys(server1.interest.patterns)).not.toContain(SUBJECT); + expect(Object.keys(server2.interest.patterns)).not.toContain(SUBJECT); + }); + + const count = 450; + let server3; + it(`one client subscribes to ${count} distinct subjects and these are all visible in the other servers -- all messages get routed properly when sent to all subjects`, async () => { + server3 = await initConatServer({ valkey }); + const v: any[] = []; + let subj; + for (let i = 0; i < count; i++) { + subj = `subject.${i}`; + v.push(client1.subscribe(subj)); + } + const subs = await Promise.all(v); + await waitForSubscription(server1, subj); + await waitForSubscription(server2, subj); + await waitForSubscription(server3, subj); + + for (let i = 0; i < count; i++) { + expect(Object.keys(server1.interest.patterns)).toContain(`subject.${i}`); + expect(Object.keys(server2.interest.patterns)).toContain(`subject.${i}`); + expect(Object.keys(server3.interest.patterns)).toContain(`subject.${i}`); + } + + // and they work: + const p: any[] = []; + const p2: any[] = []; + for (let i = 0; i < count; i++) { + p.push(client2.publish(`subject.${i}`, i)); + p2.push(subs[i].next()); + } + await Promise.all(p); + const result = await Promise.all(p2); + for (let i = 0; i < count; i++) { + expect(result[i].value.data).toBe(i); + } + + // and can unsubscribe + for (let i = 0; i < count; i++) { + subs[i].close(); + } + await waitForNonSubscription(server1, subj); + await waitForNonSubscription(server2, subj); + await waitForNonSubscription(server3, subj); + for (let i = 0; i < count; i++) { + expect(Object.keys(server1.interest.patterns)).not.toContain( + `subject.${i}`, + ); + expect(Object.keys(server2.interest.patterns)).not.toContain( + `subject.${i}`, + ); + expect(Object.keys(server3.interest.patterns)).not.toContain( + `subject.${i}`, + ); + } + }); + + it("cleans up", () => { + valkeyServer.close(); + client1.close(); + client2.close(); + server1.close(); + server2.close(); + }); +}); + +// this is very important, since the sticky resolution needs to be consistent +describe("create two servers connected via valkey, and verify that *sticky* subs properly work", () => { + let server1, server2, valkey, valkeyServer, client1, client2; + it("creates valkey, two servers and two clients", async () => { + valkeyServer = await runValkey(); + valkey = valkeyServer.address; + server1 = await initConatServer({ valkey }); + client1 = server1.client(); + server2 = await initConatServer({ valkey }); + client2 = server2.client(); + }); + + let s1, s2, stickyTarget; + const pattern = "sticky.io.*"; + it("setup a sticky server on both clients, then observe that its state is consistent", async () => { + s1 = await client1.subscribe(pattern, { queue: STICKY_QUEUE_GROUP }); + s2 = await client2.subscribe(pattern, { queue: STICKY_QUEUE_GROUP }); + await waitForSubscription(server1, pattern); + await waitForSubscription(server2, pattern); + (async () => { + for await (const x of s1) { + x.respond("s1"); + } + })(); + (async () => { + for await (const x of s2) { + x.respond("s2"); + } + })(); + + // we select a specific subject sticky.io.foo that matches the pattern : + const x = await client1.request("sticky.io.foo", null); + await waitForSticky(server1, "sticky.io.*"); + await waitForSticky(server2, "sticky.io.*"); + // this is the server it ended up hitting. + stickyTarget = x.data; + // check it still does + for (let i = 0; i < 3; i++) { + const y = await client1.request("sticky.io.foo", null); + expect(y.data).toBe(stickyTarget); + } + await waitForSticky(server2, "sticky.io.*"); + // another client requesting sticky.io.foo even though a different + // socketio conat server must get the same target: + const z = await client2.request("sticky.io.foo", null); + expect(z.data).toBe(stickyTarget); + + expect(server1.sticky).toEqual(server2.sticky); + expect(server1.sticky[pattern] != null).toBe(true); + // the last segment of the subject is discarded in the sticky choice: + expect(server1.sticky[pattern]["sticky.io"] != null).toBe(true); + }); + + let server3, server4, client3; + it("add new conat servers and observe sticky mapping is still the same so using shared state instead of consistent hashing", async () => { + server3 = await initConatServer({ valkey }); + server4 = await initConatServer({ valkey }); + await waitForSticky(server3, "sticky.io.*"); + await waitForSticky(server4, "sticky.io.*"); + await wait({ + until: () => { + return ( + server3.sticky[pattern] != null && server4.sticky[pattern] != null + ); + }, + }); + expect(server1.sticky).toEqual(server3.sticky); + expect(server1.sticky).toEqual(server4.sticky); + + client3 = server3.client(); + const z = await client3.request("sticky.io.foo", null); + expect(z.data).toBe(stickyTarget); + }); + + it("cleans up", () => { + valkeyServer.close(); + client1?.close(); + client2?.close(); + client3?.close(); + server1?.close(); + server2?.close(); + server3?.close(); + server4?.close(); + }); +}); + +afterAll(after); diff --git a/src/packages/backend/conat/test/service.test.ts b/src/packages/backend/conat/test/service.test.ts new file mode 100644 index 0000000000..aa7ea5fe9a --- /dev/null +++ b/src/packages/backend/conat/test/service.test.ts @@ -0,0 +1,280 @@ +/* + +DEVELOPMENT: + +pnpm test ./service.test.ts + +*/ + +import { callConatService, createConatService } from "@cocalc/conat/service"; +import { + createServiceClient, + createServiceHandler, +} from "@cocalc/conat/service/typed"; +import { before, after } from "@cocalc/backend/conat/test/setup"; +import { wait } from "@cocalc/backend/conat/test/util"; +import { is_date as isDate } from "@cocalc/util/misc"; +import { delay } from "awaiting"; +import { initConatServer } from "@cocalc/backend/conat/test/setup"; +import getPort from "@cocalc/backend/get-port"; +import { once } from "@cocalc/util/async-utils"; + +beforeAll(before); + +describe("create a service and test it out", () => { + let s; + it("creates a service", async () => { + s = createConatService({ + service: "echo", + handler: (mesg) => mesg.repeat(2), + }); + await once(s, "running"); + expect(await callConatService({ service: "echo", mesg: "hello" })).toBe( + "hellohello", + ); + }); + + it("closes the services and observes it doesn't work anymore", async () => { + s.close(); + await expect(async () => { + await callConatService({ service: "echo", mesg: "hi", timeout: 250 }); + }).rejects.toThrowError("timeout"); + }); +}); + +describe("verify that you can create a service AFTER calling it and things to still work fine", () => { + let result = ""; + it("call a service that does not exist yet", () => { + (async () => { + result = await callConatService({ service: "echo3", mesg: "hello " }); + })(); + }); + + it("create the echo3 service and observe that it answer the request we made before the service was created", async () => { + const s = createConatService({ + service: "echo3", + handler: (mesg) => mesg.repeat(3), + }); + await wait({ until: () => result }); + expect(result).toBe("hello hello hello "); + + s.close(); + }); +}); + +describe("create and test a more complicated service", () => { + let client, service; + it("defines the service", async () => { + interface Api { + add: (a: number, b: number) => Promise; + concat: (a: Buffer, b: Buffer) => Promise; + now: () => Promise; + big: (n: number) => Promise; + len: (s: string) => Promise; + } + + const name = "my-service"; + service = await createServiceHandler({ + service: name, + subject: name, + description: "My Service", + impl: { + // put any functions here that take/return MsgPack'able values + add: async (a, b) => a + b, + concat: async (a, b) => Buffer.concat([a, b]), + now: async () => { + await delay(5); + return new Date(); + }, + big: async (n: number) => "x".repeat(n), + len: async (s: string) => s.length, + }, + }); + + client = createServiceClient({ + service: name, + subject: name, + }); + }); + + it("tests the service", async () => { + // these calls are all type checked using typescript + expect(await client.add(2, 3)).toBe(5); + + const a = Buffer.from("hello"); + const b = Buffer.from(" conat"); + expect(await client.concat(a, b)).toEqual(Buffer.concat([a, b])); + + const d = await client.now(); + expect(isDate(d)).toBe(true); + expect(Math.abs(d.valueOf() - Date.now())).toBeLessThan(100); + + const n = 10 * 1e6; + expect((await client.big(n)).length).toBe(n); + + expect(await client.len("x".repeat(n))).toBe(n); + }); + + it("cleans up", () => { + service.close(); + }); +}); + +describe("create a service with specified client, stop and start the server, and see service still works", () => { + let server; + let client; + let client2; + let port; + it("create a conat server and client", async () => { + port = await getPort(); + server = await initConatServer({ port }); + client = server.client({ reconnectionDelay: 50 }); + client2 = server.client({ reconnectionDelay: 50 }); + }); + + let service; + it("create service using specific client and call it using both clients", async () => { + //You usually do NOT want a non-ephemeral service... + service = createConatService({ + client, + service: "double", + handler: (mesg) => mesg.repeat(2), + }); + await once(service, "running"); + + expect( + await callConatService({ client, service: "double", mesg: "hello" }), + ).toBe("hellohello"); + + expect( + await callConatService({ + client: client2, + service: "double", + mesg: "hello", + }), + ).toBe("hellohello"); + }); + + it("disconnect client and check service still works on reconnect", async () => { + // cause a disconnect -- client will connect again in 50ms soon + // and then handle the request below: + client.conn.io.engine.close(); + await delay(100); + expect( + await callConatService({ + client: client2, + service: "double", + mesg: "hello", + }), + ).toBe("hellohello"); + }); + + it("disconnect client2 and check service still works on reconnect", async () => { + // cause a disconnect -- client will connect again in 50ms soon + // and handle the request below: + client2.conn.io.engine.close(); + await delay(100); + expect( + await callConatService({ + client: client2, + service: "double", + mesg: "hello", + }), + ).toBe("hellohello"); + }); + + it("disconnect both clients and check service still works on reconnect", async () => { + // cause a disconnect -- client will connect again in 50ms soon + // and handle the request below: + client.conn.io.engine.close(); + client2.conn.io.engine.close(); + expect( + await callConatService({ + client: client2, + service: "double", + mesg: "hello", + }), + ).toBe("hellohello"); + }); + + it("kills the server, then makes another server serving on the same port", async () => { + await server.close(); + await delay(250); + server = await initConatServer({ port }); + // Killing the server is not at all a normal thing to expect, and causes loss of + // its state. The clients have to sync realize subscriptions are missing. This + // takes a fraction of a second and the call below won't immediately work. + await wait({ + until: async () => { + try { + await callConatService({ + client: client2, + service: "double", + mesg: "hello", + noRetry: true, + timeout: 250, + }); + return true; + } catch (err) { + return false; + } + }, + }); + expect( + await callConatService({ + client: client2, + service: "double", + mesg: "hello", + noRetry: true, + }), + ).toBe("hellohello"); + }); + + it("cleans up", () => { + service.close(); + client.close(); + client2.close(); + server.close(); + }); +}); + +describe("create a slow service and check that the timeout parameter works", () => { + let s; + it("creates a slow service", async () => { + s = createConatService({ + service: "slow", + handler: async (d) => { + await delay(d); + return { delay: d }; + }, + }); + await once(s, "running"); + }); + + it("confirms it works", async () => { + const t0 = Date.now(); + const r = await callConatService({ + service: s.name, + mesg: 50, + }); + expect(r).toEqual({ delay: 50 }); + expect(Date.now() - t0).toBeGreaterThan(45); + expect(Date.now() - t0).toBeLessThan(500); + }); + + it("confirms it throws a timeout error", async () => { + await expect(async () => { + await callConatService({ + service: s.name, + mesg: 5000, + timeout: 75, + }); + }).rejects.toThrowError("imeout"); + }); + + it("clean up", async () => { + s.close(); + }); +}); + +afterAll(after); diff --git a/src/packages/backend/conat/test/setup.ts b/src/packages/backend/conat/test/setup.ts new file mode 100644 index 0000000000..b413075291 --- /dev/null +++ b/src/packages/backend/conat/test/setup.ts @@ -0,0 +1,160 @@ +import getPort from "@cocalc/backend/get-port"; +import { type Client } from "@cocalc/conat/core/client"; +import { + init as createConatServer, + type Options, + type ConatServer, +} from "@cocalc/conat/core/server"; +import getLogger from "@cocalc/backend/logger"; +import { setConatClient } from "@cocalc/conat/client"; +import { server as createPersistServer } from "@cocalc/backend/conat/persist"; +import { syncFiles } from "@cocalc/conat/persist/context"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "path"; +export { wait } from "@cocalc/backend/conat/test/util"; +export { delay } from "awaiting"; +export { setDefaultTimeouts } from "@cocalc/conat/core/client"; +export { once } from "@cocalc/util/async-utils"; +import { spawn, ChildProcess } from "node:child_process"; + +const logger = getLogger("conat:test:setup"); + +export const path = "/conat"; + +export async function initConatServer( + options: Partial = {}, +): Promise { + logger.debug("init"); + if (!options?.port) { + const port = await getPort(); + options = { ...options, port }; + } + + return createConatServer(options); +} + +export let tempDir; +export let server: any = null; +export let persistServer: any = null; + +export async function createServer(opts?) { + const port = await getPort(); + server = await initConatServer({ port, path, ...opts }); + return server; +} + +export async function restartServer() { + const port = server.options.port; + await server.close(); + await createServer({ port }); +} + +export async function restartPersistServer() { + await persistServer.close(); + client = connect(); + persistServer = createPersistServer({ client }); +} + +// one pre-made client +export let client; +export async function before() { + tempDir = await mkdtemp(join(tmpdir(), "conat-test")); + server = await createServer(); + client = connect(); + persistServer = createPersistServer({ client }); + syncFiles.local = join(tempDir, "local"); + syncFiles.archive = join(tempDir, "archive"); + setConatClient({ + conat: connect, + getLogger, + }); +} + +const clients: Client[] = []; +export function connect(opts?): Client { + const cn = server.client({ noCache: true, ...opts }); + clients.push(cn); + return cn; +} + +export async function after() { + persistServer?.close(); + await rm(tempDir, { force: true, recursive: true }); + await server?.close(); + for (const cn of clients) { + cn.close(); + } + for (const { close } of valkeys) { + close(); + } +} + +process.once("exit", () => { + for (const { close } of valkeys) { + try { + close(); + } catch {} + } +}); +["SIGINT", "SIGTERM", "SIGQUIT"].forEach((sig) => { + process.once(sig, () => { + process.exit(); + }); +}); + +// runs a new ephemeral valkey server on an available port, +// returning that port. +const valkeys: any[] = []; +export async function runValkey(): Promise<{ + port: number; + address: string; + close: () => void; + password: string; +}> { + const port = await getPort(); + const password = "testpass"; + + // sapwn valkey-server listening on port running in a mode where + // data is never saved to disk using the nodejs spawn command: + // // Start valkey-server with in-memory only, no persistence + const child: ChildProcess = spawn( + "valkey-server", + [ + "--port", + String(port), + "--save", + "", + "--appendonly", + "no", + "--requirepass", + password, + ], + { + stdio: "ignore", // or "inherit" for debugging + detached: false, + }, + ); + + let closed = false; + const close = () => { + if (closed) return; + closed = true; + if (!child?.pid) return; + try { + process.kill(child.pid, "SIGKILL"); + } catch { + // already dead or not found + } + }; + + const server = { + port, + close, + address: `valkey://:${password}@localhost:${port}`, + password, + }; + valkeys.push(server); + + return server; +} diff --git a/src/packages/backend/conat/test/socket/basic.test.ts b/src/packages/backend/conat/test/socket/basic.test.ts new file mode 100644 index 0000000000..d4a19d80bf --- /dev/null +++ b/src/packages/backend/conat/test/socket/basic.test.ts @@ -0,0 +1,701 @@ +/* + +pnpm test `pwd`/basic.test.ts + +*/ + +import { + before, + after, + connect, + wait, + setDefaultTimeouts, +} from "@cocalc/backend/conat/test/setup"; +import { once } from "@cocalc/util/async-utils"; +import { delay } from "awaiting"; + +beforeAll(async () => { + await before(); + setDefaultTimeouts({ request: 750, publish: 750 }); +}); + +describe("create a server and client, then send a message and get a response", () => { + let client, + server, + cn1, + cn2, + subject = "response.double"; + + it("creates the client and server", () => { + cn1 = connect(); + server = cn1.socket.listen(subject); + server.on("connection", (socket) => { + socket.on("data", (data) => { + socket.write(`${data}`.repeat(2)); + }); + }); + }); + + it("connects as client and tests out the server", async () => { + cn2 = connect(); + client = cn2.socket.connect(subject); + client.write("cocalc"); + const [data] = await once(client, "data"); + expect(data).toBe("cocalccocalc"); + }); + + it("send 3 messages and get 3 responses, in order", async () => { + client.write("a"); + client.write("b"); + client.write("c"); + expect((await once(client, "data"))[0]).toBe("aa"); + expect((await once(client, "data"))[0]).toBe("bb"); + expect((await once(client, "data"))[0]).toBe("cc"); + }); + + const count = 250; + it(`sends ${count} messages and gets responses, so its obviously not super slow`, async () => { + const t = Date.now(); + for (let i = 0; i < count; i++) { + client.write(`${i}`); + } + for (let i = 0; i < count; i++) { + expect((await once(client, "data"))[0]).toBe(`${i}`.repeat(2)); + } + expect(Date.now() - t).toBeLessThan(5000); + }); + + it("cleans up", () => { + client.close(); + server.close(); + cn1.close(); + cn2.close(); + }); +}); + +describe("create a client first, then the server, and see that write still works (testing the order); also include headers in both directions.", () => { + let client, server, cn1, cn2, requestPromise; + const subject = "cocalc-order"; + + it("connects as client and writes to the server that doesn't exist yet", async () => { + cn2 = connect(); + client = cn2.socket.connect(subject); + client.write("cocalc", { headers: { my: "header" } }); + }); + + it("we fire off a request as well, but don't wait for it", () => { + requestPromise = client.request("foo"); + }); + + it("creates the server", () => { + cn1 = connect(); + server = cn1.socket.listen(subject); + server.on("connection", (socket) => { + socket.on("data", (data, headers) => { + socket.write(`${data}`.repeat(2), { headers }); + }); + socket.on("request", (mesg) => { + mesg.respondSync("bar", { headers: "x" }); + }); + }); + }); + + it("it still works out", async () => { + const [data, headers] = await once(client, "data"); + expect(data).toBe("cocalccocalc"); + expect(headers).toEqual({ my: "header" }); + }); + + it("get back the response from the request we created above", async () => { + const response = await requestPromise; + expect(response.data).toBe("bar"); + expect(response.headers).toBe("x"); + }); + + it("cleans up", () => { + client.close(); + server.close(); + cn1.close(); + cn2.close(); + }); +}); + +describe("create a client first and write more messages than the queue size results in an error", () => { + let client, server, cn1, cn2; + const subject = "conat.too.many.messages"; + + let count = 5, + maxQueueSize = 3, + iter; + it("connects as client with a small queue and fill it", async () => { + cn2 = connect(); + let fails = 0; + client = cn2.socket.connect(subject, { maxQueueSize }); + iter = client.iter(); + for (let i = 0; i < count; i++) { + try { + client.write(`${i}`); + } catch (err) { + // should fail for i=4,5 + expect(i).toBeGreaterThan(count - maxQueueSize); + fails += 1; + } + } + expect(fails).toBe(2); + expect(client.queuedWrites.length).toBe(3); + }); + + const serverRecv: any[] = []; + let serverSocket; + it("creates the server", () => { + cn1 = connect(); + server = cn1.socket.listen(subject, { maxQueueSize }); + server.on("connection", (socket) => { + serverSocket = socket; + socket.on("data", (data) => { + serverRecv.push(data); + socket.write(`${data}`.repeat(2)); + }); + }); + }); + + it(`first ${maxQueueSize} messages do get sent`, async () => { + for (let i = 0; i < maxQueueSize; i++) { + const { value } = await iter.next(); + expect(value[0]).toBe(`${i}`.repeat(2)); + } + expect(serverRecv).toEqual(["0", "1", "2"]); + }); + + it("wait for client to drain; then we can now send another message without an error", async () => { + await client.waitUntilDrain(); + client.write("foo"); + }); + + it("writing too many messages to the server socket also fails", async () => { + if (serverSocket.tcp.send.unsent > 0) { + await once(serverSocket, "drain"); + } + expect(serverSocket.tcp.send.unsent).toBe(0); + serverSocket.write(0); + serverSocket.write(1); + serverSocket.write(2); + expect(() => serverSocket.write(3)).toThrow("WRITE FAILED"); + try { + serverSocket.write(4); + } catch (err) { + expect(err.code).toBe("ENOBUFS"); + } + }); + + it("cleans up", () => { + client.close(); + server.close(); + cn1.close(); + cn2.close(); + }); +}); + +describe("test having two clients and see that communication is independent and also broadcast to both", () => { + let client1, client2, server, cn1, cn2, cn3; + + it("creates a server and two clients", async () => { + cn3 = connect(); + server = cn3.socket.listen("cocalc2"); + server.on("connection", (socket) => { + socket.on("data", (data) => { + socket.write(`${data}`.repeat(2)); + }); + }); + + cn1 = connect(); + client1 = cn1.socket.connect("cocalc2"); + cn2 = connect(); + client2 = cn2.socket.connect("cocalc2"); + }); + + it("each client uses the server separately", async () => { + const x1 = once(client1, "data"); + const x2 = once(client2, "data"); + client1.write("one"); + client2.write("two"); + expect((await x1)[0]).toBe("oneone"); + expect((await x2)[0]).toBe("twotwo"); + }); + + it("server broadcast to all clients", async () => { + const x1 = once(client1, "data"); + const x2 = once(client2, "data"); + server.write("broadcast"); + expect((await x1)[0]).toBe("broadcast"); + expect((await x2)[0]).toBe("broadcast"); + }); + + it("test with a channel", async () => { + const s1 = server.channel("one"); + const c1 = client1.channel("one"); + const c2 = client2.channel("one"); + s1.on("connection", (socket) => { + socket.on("data", (data) => { + socket.write(`1${data}`); + }); + }); + const x1 = once(c1, "data"); + const x2 = once(c2, "data"); + c1.write("c1"); + expect((await x1)[0]).toBe("1c1"); + c2.write("c2"); + expect((await x2)[0]).toBe("1c2"); + + s1.close(); + c1.close(); + c2.close(); + }); + + it("cleans up", () => { + client1.close(); + client2.close(); + server.close(); + cn1.close(); + cn2.close(); + cn3.close(); + }); +}); + +describe("create a server and client. Disconnect the client and see from the server point of view that it disconnected.", () => { + let server, cn1; + + it("creates the server", () => { + cn1 = connect(); + server = cn1.socket.listen("disconnect.io"); + server.on("connection", (socket) => { + socket.on("data", () => { + socket.write(`clients=${Object.keys(server.sockets).length}`); + }); + }); + expect(Object.keys(server.sockets).length).toBe(0); + }); + + let client; + it("connects with a client", async () => { + cn1 = connect(); + client = cn1.socket.connect("disconnect.io"); + const r = once(client, "data"); + client.write("hello"); + expect((await r)[0]).toBe("clients=1"); + }); + + it("disconnects and sees the count of clients goes back to 0", async () => { + client.close(); + await wait({ + until: () => { + return Object.keys(server.sockets).length == 0; + }, + }); + }); + + it("creates a new client, connects to server, then closes the server and the client sees that and closes.", async () => { + client = cn1.socket.connect("disconnect.io"); + const iter = client.iter(); + // confirm working: + client.write("hello"); + const { value } = await iter.next(); + expect(value[0]).toBe("clients=1"); + + expect(client.state).toBe("ready"); + const closed = once(client, "closed"); + // now close server and wait for state to quickly automatically + // switch to not ready anymore + const t0 = Date.now(); + server.close(); + await closed; + expect(Date.now() - t0).toBeLessThan(250); + }); +}); + +describe("create two socket servers with the same subject to test that sockets are sticky", () => { + const subject = "a.sticks.place"; + let c1, c2, s1, s2; + it("creates two distinct socket servers with the same subject", () => { + c1 = connect(); + c2 = connect(); + s1 = c1.socket.listen(subject); + s1.on("connection", (socket) => { + // console.log("s1 got connection"); + socket.on("data", () => { + // console.log("s1 got data"); + socket.write("s1"); + }); + socket.on("request", (mesg) => mesg.respond("s1")); + }); + s2 = c2.socket.listen(subject); + s2.on("connection", (socket) => { + // console.log("s2 got connection"); + socket.on("data", () => { + // console.log("s2 got data"); + socket.write("s2"); + }); + socket.on("request", (mesg) => mesg.respond("s2")); + }); + }); + + let c3, client, resp; + it("creates a client and verifies writes all go to the same server", async () => { + c3 = connect(); + client = c3.socket.connect(subject); + const iter = client.iter(); + client.write(null); + const { value } = await iter.next(); + resp = value[0]; + // all additional messages end up going to the same server, because + // of "sticky" subscriptions :-) + for (let i = 0; i < 25; i++) { + client.write(null); + const { value: value1 } = await iter.next(); + expect(resp).toBe(value1[0]); + } + }); + + let c3b, s3; + it("add one more server and verify that messages still all go to the right place", async () => { + c3b = connect(); + s3 = c3b.socket.listen(subject); + let newServerGotConnection = false; + s3.on("connection", (socket) => { + //console.log("s3 got a connection"); + newServerGotConnection = true; + socket.on("data", () => { + //console.log("s3 got data", { data }); + socket.write("s3"); + }); + }); + const iter = client.iter(); + for (let i = 0; i < 25; i++) { + client.write(null); + const { value: value1 } = await iter.next(); + if (resp != value1[0]) { + throw Error("sticky load balancing failed!?"); + } + } + expect(newServerGotConnection).toBe(false); // redundant... + }); + + it("also verify that request/reply messaging go to the right place (stickiness works the same way)", async () => { + for (let i = 0; i < 25; i++) { + const x = await client.request(null); + expect(x.data).toBe(resp); + } + }); + + it("remove the server we're connected to and see that the client socket closes, since all state on the other end is gone (this is the only possible thing that should happen!)", async () => { + if (resp == "s1") { + s1.close(); + } else if (resp == "s2") { + s2.close(); + } + await once(client, "closed"); + }); + + it("cleans up", () => { + s1.close(); + s2.close(); + s3.close(); + c1.close(); + c2.close(); + c3.close(); + c3b.close(); + client.close(); + }); +}); + +describe("create a server where the subject has a wildcard, so clients can e.g., authentication themselves by having permission to write to the subject", () => { + let client, server, cn1, cn2; + it("creates the client and server", () => { + cn1 = connect(); + server = cn1.socket.listen("changefeeds.*"); + server.on("connection", (socket) => { + socket.on("data", () => { + socket.write(socket.subject.split(".")[1]); + }); + }); + }); + + it("connects as client on different matching subjects", async () => { + cn2 = connect(); + client = cn2.socket.connect("changefeeds.account-5077"); + const x = once(client, "data"); + client.write(null); + const [data] = await x; + expect(data).toBe("account-5077"); + client.close(); + + client = cn2.socket.connect("changefeeds.account-389"); + const x2 = once(client, "data"); + client.write(null); + const [data2] = await x2; + expect(data2).toBe("account-389"); + }); + + it("cleans up", () => { + client.close(); + server.close(); + cn1.close(); + cn2.close(); + }); +}); + +describe("Check that the automatic reconnection parameter works", () => { + let server, cn1; + it("creates the server", () => { + cn1 = connect(); + server = cn1.socket.listen("recon"); + server.on("connection", (socket) => { + socket.on("data", (data) => { + socket.write(data); + }); + }); + }); + + it("create a client with reconnection (the default) and confirm it works (all states hit)", async () => { + const socket = cn1.socket.connect("recon"); + expect(socket.reconnection).toBe(true); // the default + await once(socket, "ready"); + // have to listen before we trigger it: + const y = once(socket, "disconnected"); + const x = once(socket, "connecting"); + socket.disconnect(); + const z = once(socket, "data"); + + // write when not connected -- this should get sent + // when we connect: + socket.write("hi"); + + await once(socket, "ready"); + await y; + await x; + expect((await z)[0]).toBe("hi"); + socket.close(); + }); + + it("creates a client without reconnection", async () => { + const socket = cn1.socket.connect("recon", { reconnection: false }); + expect(socket.reconnection).toBe(false); + await once(socket, "ready"); + socket.disconnect(); + await delay(50); + // still disconnected + expect(socket.state).toBe("disconnected"); + // but we can manually connect + socket.connect(); + await once(socket, "ready"); + socket.close(); + }); +}); + +describe("creating multiple sockets from the one client to one server works (they should be distinct)", () => { + let server, cn1, cn2; + const subject = "multiple.sockets.edu"; + it("creates the client and server", () => { + cn1 = connect(); + server = cn1.socket.listen(subject); + server.on("connection", (socket) => { + socket.on("data", (data) => { + socket.write(`${data}-${socket.id}`); + }); + }); + }); + + it("creates two client sockets", async () => { + cn2 = connect(); + const socket1 = cn2.socket.connect(subject); + const socket2 = cn2.socket.connect(subject); + expect(socket1.id).not.toEqual(socket2.id); + const x = once(socket1, "data"); + const y = once(socket2, "data"); + socket1.write("cocalc"); + socket2.write("conat"); + const [data] = await x; + expect(data).toBe(`cocalc-${socket1.id}`); + const [data2] = await y; + expect(data2).toBe(`conat-${socket2.id}`); + const x1 = once(socket1, "data"); + const y1 = once(socket2, "data"); + + // also test broadcast + server.write("hello"); + expect((await x1)[0]).toBe("hello"); + expect((await y1)[0]).toBe("hello"); + + socket1.close(); + socket2.close(); + }); + + it("cleans up", () => { + server.close(); + cn1.close(); + cn2.close(); + }); +}); + +describe("test request/respond from client to server and from server to client", () => { + let socket1, socket2, server, cn1, cn2, cn3; + const subject = "request-respond-demo"; + const sockets: any[] = []; + + it("creates a server and two sockets", async () => { + cn3 = connect(); + server = cn3.socket.listen(subject); + server.on("connection", (socket) => { + sockets.push(socket); + socket.on("request", (mesg) => { + mesg.respond(`hi ${mesg.data}, from server`); + }); + }); + + cn1 = connect(); + socket1 = cn1.socket.connect(subject); + socket1.on("request", (mesg) => { + mesg.respond(`hi ${mesg.data}, from socket1`); + }); + + cn2 = connect(); + socket2 = cn2.socket.connect(subject); + socket2.on("request", (mesg) => { + mesg.respond(`hi ${mesg.data}, from socket2`); + }); + }); + + it("each socket calls the server", async () => { + expect((await socket1.request("socket1")).data).toBe( + "hi socket1, from server", + ); + expect((await socket2.request("socket2")).data).toBe( + "hi socket2, from server", + ); + }); + + it("the server individually calls each socket", async () => { + // note that sockets[0] and sockets[1] might be in + // either order. + const x = (await sockets[0].request("server")).data; + const y = (await sockets[1].request("server")).data; + expect(x).not.toEqual(y); + expect(x).toContain("hi server, from socket"); + expect(y).toContain("hi server, from socket"); + }); + + it("broadcast a request to all connected sockets", async () => { + const v = (await server.request("server")) as any; + const w = v.map((y: any) => y.data); + const S = new Set(["hi server, from socket1", "hi server, from socket2"]); + expect(new Set(w)).toEqual(S); + + // also broadcast and use race, so we get just the first response. + const x = await server.request("server", { race: true }); + expect(S.has(x.data)).toBe(true); + }); + + it("cleans up", () => { + socket1.close(); + socket2.close(); + server.close(); + cn1.close(); + cn2.close(); + cn3.close(); + }); +}); + +describe("test request/respond with headers", () => { + let socket1, + server, + cn1, + cn2, + sockets: any[] = []; + const subject = "request-respond-headers"; + + it("creates a server and a socket", async () => { + cn2 = connect(); + server = cn2.socket.listen(subject); + server.on("connection", (socket) => { + sockets.push(socket); + socket.on("request", (mesg) => { + mesg.respond(`server: ${mesg.data}`, { + headers: { ...mesg.headers, server: true }, + }); + }); + }); + + cn1 = connect(); + socket1 = cn1.socket.connect(subject); + socket1.on("request", (mesg) => { + mesg.respond(`socket1: ${mesg.data}`, { + headers: { ...mesg.headers, socket1: true }, + }); + }); + }); + + it("headers work when client calls server", async () => { + const x = await socket1.request("hi", { headers: { foo: 10 } }); + expect(x.data).toBe("server: hi"); + expect(x.headers).toEqual( + expect.objectContaining({ foo: 10, server: true }), + ); + }); + + it("headers work when server calls client", async () => { + const x = await sockets[0].request("hi", { headers: { foo: 10 } }); + expect(x.data).toBe("socket1: hi"); + expect(x.headers).toEqual( + expect.objectContaining({ foo: 10, socket1: true }), + ); + }); + + it("cleans up", () => { + socket1.close(); + server.close(); + cn1.close(); + cn2.close(); + }); +}); + +describe("test requestMany/respond", () => { + let socket1, + server, + cn1, + cn2, + sockets: any[] = []; + const subject = "requestMany"; + + it("creates a server that handles a requestMany, and a client", async () => { + cn2 = connect(); + server = cn2.socket.listen(subject); + server.on("connection", (socket) => { + sockets.push(socket); + socket.on("request", (mesg) => { + for (let i = 0; i < mesg.data; i++) { + mesg.respond(i); + } + }); + }); + + cn1 = connect(); + socket1 = cn1.socket.connect(subject); + }); + + it("sends a requestMany request and get 3 responses", async () => { + const sub = await socket1.requestMany(10); + for (let i = 0; i < 10; i++) { + expect((await sub.next()).value.data).toBe(i); + } + sub.close(); + }); + + it("cleans up", () => { + socket1.close(); + server.close(); + cn1.close(); + cn2.close(); + }); +}); + +afterAll(after); diff --git a/src/packages/backend/conat/test/socket/keepalive.test.ts b/src/packages/backend/conat/test/socket/keepalive.test.ts new file mode 100644 index 0000000000..02540f723f --- /dev/null +++ b/src/packages/backend/conat/test/socket/keepalive.test.ts @@ -0,0 +1,99 @@ +/* +pnpm test ./keepalive.test.ts +*/ + +import { before, after, connect, wait } from "@cocalc/backend/conat/test/setup"; +import { delay } from "awaiting"; + +beforeAll(before); + +describe("test a server with a short keepalive time", () => { + let client, + server, + cn1, + cn2, + sockets: any[] = []; + + const keepAlive = 100; + const keepAliveTimeout = 50; + + it("creates a socket server with very short keepalive", async () => { + cn1 = connect(); + server = cn1.socket.listen("keepalive-server.com", { + keepAlive, + keepAliveTimeout, + }); + server.on("connection", (socket) => { + sockets.push(socket); + }); + expect(server.keepAlive).toBe(keepAlive); + expect(server.keepAliveTimeout).toBe(keepAliveTimeout); + cn2 = connect(); + client = cn2.socket.connect("keepalive-server.com", { + keepAlive: 10000, + keepAliveTimeout: 10000, + reconnection: false, + }); + }); + + it("waits twice the keepAlive time and observes time was updated and sockets still alive", async () => { + await delay(4 * keepAlive); + expect(sockets[0].state).toBe("ready"); + expect(Math.abs(sockets[0].alive.last - Date.now())).toBeLessThan( + 1.2 * (keepAlive + keepAliveTimeout), + ); + }); + + it("breaks the client side of the socket and observes the server automatically disconnects", async () => { + client.sub.close(); + await delay(1.2 * (keepAlive + keepAliveTimeout)); + expect(sockets[0].state).toBe("closed"); + }); +}); + +describe.only("test a client with a short keepalive time", () => { + let client, + server, + cn1, + cn2, + sockets: any[] = []; + + const keepAlive = 100; + const keepAliveTimeout = 50; + + it("creates a socket server with long keepalive and client with a very short one", async () => { + cn1 = connect(); + server = cn1.socket.listen("keepalive-client.com", { + keepAlive: 10000, + keepAliveTimeout: 10000, + }); + server.on("connection", (socket) => { + sockets.push(socket); + }); + cn2 = connect(); + client = cn2.socket.connect("keepalive-client.com", { + keepAlive, + keepAliveTimeout, + reconnection: false, + }); + expect(client.keepAlive).toBe(keepAlive); + expect(client.keepAliveTimeout).toBe(keepAliveTimeout); + }); + + it("waits several times the keepAlive time and observes time was updated and sockets still alive", async () => { + await delay(4 * keepAlive); + expect(client.state).toBe("ready"); + expect(Math.abs(client.alive.last - Date.now())).toBeLessThan( + keepAlive + keepAliveTimeout, + ); + }); + + it("breaks the server side of the socket and observes the client automatically disconnects quickly", async () => { + // hack to make server /dev/null any command from client + server.handleCommandFromClient = () => {}; + await wait({ until: () => client.state == "disconnected" }); + expect(client.state).toBe("disconnected"); + }); +}); + +afterAll(after); diff --git a/src/packages/backend/conat/test/socket/restarts.test.ts b/src/packages/backend/conat/test/socket/restarts.test.ts new file mode 100644 index 0000000000..456e062023 --- /dev/null +++ b/src/packages/backend/conat/test/socket/restarts.test.ts @@ -0,0 +1,184 @@ +/* + +pnpm test `pwd`/restarts.test.ts + +*/ + +import { + before, + after, + connect, + restartServer, + setDefaultTimeouts, +} from "@cocalc/backend/conat/test/setup"; +import { once } from "@cocalc/util/async-utils"; + +beforeAll(async () => { + await before(); + setDefaultTimeouts({ request: 750, publish: 750 }); +}); + +jest.setTimeout(15000); + +describe("create a client and server and socket, verify it works, restart conat server, then confirm that socket still works", () => { + const SUBJECT = "reconnect.one"; + + let client, + server, + cn1, + cn2, + sockets: any[] = []; + + it("creates the client and server and confirms it works", async () => { + cn1 = connect(); + server = cn1.socket.listen(SUBJECT); + server.on("connection", (socket) => { + sockets.push(socket); + socket.on("data", (data) => { + socket.write(`${data}`.repeat(2)); + }); + socket.on("request", (mesg) => { + mesg.respond("hello"); + }); + }); + cn2 = connect(); + client = cn2.socket.connect(SUBJECT); + + const iter = client.iter(); + client.write("cocalc"); + const { value } = await iter.next(); + expect(value[0]).toBe("cocalccocalc"); + + expect((await client.request(null)).data).toBe("hello"); + }); + + async function waitForClientsToReconnect() { + await Promise.all([once(cn1, "connected"), once(cn2, "connected")]); + } + + it("restarts the conat socketio server, wait for clients to reconnect, and test sending data over socket", async () => { + await restartServer(); + await waitForClientsToReconnect(); + // sending data over socket + const iter = client.iter(); + client.write("test"); + const { value, done } = await iter.next(); + expect(done).toBe(false); + expect(value[0]).toBe("testtest"); + }); + + let socketDisconnects: string[] = []; + it("also request/respond immediately works", async () => { + expect((await client.request(null)).data).toBe("hello"); + }); + + it("observes the socket did not disconnect - they never do until a timeout or being explicitly closed, which is the point of sockets -- they are robust to client connection state", async () => { + expect(socketDisconnects.length).toBe(0); + }); + + // this test should take several seconds due to having to missed-packet detection logic + it("restart connection right when message is being sent; dropped message eventually gets through automatically without waiting for reconnect", async () => { + const iter = client.iter(); + client.write("conat "); + await restartServer(); + const { value } = await iter.next(); + expect(value[0]).toBe("conat conat "); + }); + + it("cleans up", () => { + cn1.close(); + cn2.close(); + }); +}); + +describe("test of socket and restarting server -- restart while sending data from server to the client", () => { + const SUBJECT = "reconnect.two"; + + let client, + server, + cn1, + cn2, + sockets: any[] = []; + + it("creates the client and server and confirms it works", async () => { + cn1 = connect(); + server = cn1.socket.listen(SUBJECT); + server.on("connection", (socket) => { + sockets.push(socket); + socket.on("data", (data) => { + socket.write(`${data}`.repeat(2)); + }); + }); + cn2 = connect(); + client = cn2.socket.connect(SUBJECT); + const iter = client.iter(); + client.write("cocalc"); + const { value } = await iter.next(); + expect(value[0]).toBe("cocalccocalc"); + }); + + // this test should take several seconds due to having to missed-packet detection logic + it("restart connection as we are sending data from the server to the client, and see again that nothing is lost - this the server --> client direction of the tests below which was client --> server", async () => { + const socket = sockets[0]; + const iter = client.iter(); + socket.write("sneaky"); + await restartServer(); + const { value } = await iter.next(); + expect(value[0]).toBe("sneaky"); + }); + + it("cleans up", () => { + cn1.close(); + cn2.close(); + }); +}); + +describe("another restart test: sending data while reconnecting to try to screw with order of arrival of messages", () => { + const SUBJECT = "reconnect.three"; + + let client, + server, + cn1, + cn2, + sockets: any[] = [], + iter; + it("creates the client and server and confirms it works", async () => { + cn1 = connect(); + server = cn1.socket.listen(SUBJECT); + server.on("connection", (socket) => { + sockets.push(socket); + socket.on("data", (data) => { + socket.write(`${data}`.repeat(2)); + }); + }); + cn2 = connect(); + client = cn2.socket.connect(SUBJECT); + iter = client.iter(); + client.write("one "); + const { value } = await iter.next(); + expect(value[0]).toBe("one one "); + }); + + it("now the **HARD CASE**; we do the same as above, but kill the server exactly as the message is being sent, so it is dropped", async () => { + client.write("four "); + await restartServer(); + + // write another message to socket to cause out of order message deliver + // to the other end + client.write("five "); + const { value } = await iter.next(); + expect(value[0]).toBe("four four "); + + // also checking ordering is correct too -- we next + // next get the foofoo response; + const { value: value1 } = await iter.next(); + expect(value1[0]).toBe("five five "); + }); + + it("cleans up", () => { + cn1.close(); + cn2.close(); + }); +}); + +afterAll(after); diff --git a/src/packages/backend/nats/test/sync/akv.test.ts b/src/packages/backend/conat/test/sync/akv.test.ts similarity index 63% rename from src/packages/backend/nats/test/sync/akv.test.ts rename to src/packages/backend/conat/test/sync/akv.test.ts index 193ed44e35..f3571fd4a2 100644 --- a/src/packages/backend/nats/test/sync/akv.test.ts +++ b/src/packages/backend/conat/test/sync/akv.test.ts @@ -3,20 +3,23 @@ Testing basic ops with dkv DEVELOPMENT: -pnpm exec jest --forceExit "akv.test.ts" +pnpm test ./akv.test.ts */ -import { dkv as createDkv, akv as createAkv } from "@cocalc/backend/nats/sync"; -import { once } from "@cocalc/util/async-utils"; -import { getMaxPayload } from "@cocalc/nats/util"; +import { dkv as createDkv, akv as createAkv } from "@cocalc/backend/conat/sync"; +import { wait } from "@cocalc/backend/conat/test/util"; +import { before, after, connect } from "@cocalc/backend/conat/test/setup"; + +beforeAll(before); describe("test basics with an akv", () => { - let kv; + let kv, client; const name = `test-${Math.random()}`; it("creates the akv, then set and read a value", async () => { - kv = createAkv({ name }); + client = connect(); + kv = createAkv({ name, client }); await kv.set("x", 10); expect(await kv.get("x")).toBe(10); }); @@ -30,25 +33,30 @@ describe("test basics with an akv", () => { expect(await kv.get("y")).toBe(null); }); + it("gets all keys", async () => { + expect(await kv.keys()).toEqual(["x", "y"]); + }); + it("check that deleting a value works", async () => { await kv.delete("x"); expect(await kv.get("x")).toBe(undefined); }); it("cleans up", async () => { - const k = await createDkv({ name }); + const k = await createDkv({ name, client }); k.clear(); await k.close(); }); }); describe("test interop with a dkv", () => { - let akv, dkv; + let akv, dkv, client; const name = `test-${Math.random()}`; it("creates the akv and dkv", async () => { - akv = createAkv({ name }); - dkv = await createDkv({ name }); + client = connect(); + akv = createAkv({ name, client }); + dkv = await createDkv({ name, client }); }); it("sets value in the dkv and reads it using the akv", async () => { @@ -59,9 +67,7 @@ describe("test interop with a dkv", () => { it("sets value in the akv and reads it using the dkv", async () => { await akv.set("z", 389); - if (!dkv.get("z")) { - await once(dkv, "change"); - } + await wait({ until: () => dkv.has("z") }); expect(await dkv.get("z")).toBe(389); }); @@ -76,14 +82,23 @@ describe("test interop with a dkv", () => { expect(await akv.headers("h2")).toEqual( expect.objectContaining({ foo: "baz" }), ); - if (dkv.get("h2") === undefined) { - await once(dkv, "change"); - } + await wait({ until: () => dkv.has("h2") }); expect(await dkv.headers("h2")).toEqual( expect.objectContaining({ foo: "baz" }), ); }); + it("check sqlite query fails", async () => { + await expect(async () => { + await akv.sqlite("SELECT count(*) AS n FROM messages"); + }).rejects.toThrowError("sqlite command not currently supported"); + }); + + // it("check sqlite query works", async () => { + // const v = await akv.sqlite("SELECT count(*) AS n FROM messages"); + // expect(v[0].n).toBe((await akv.keys()).length); + // }); + it("cleans up", async () => { dkv.clear(); await dkv.close(); @@ -92,24 +107,29 @@ describe("test interop with a dkv", () => { describe("testing writing and reading chunked data", () => { let maxPayload = 0; + let client; it("sanity check on the max payload", async () => { - maxPayload = await getMaxPayload(); - expect(maxPayload).toBeGreaterThan(1000000); + client = connect(); + await wait({ until: () => client.info != null }); + maxPayload = client.info?.max_payload ?? 0; + expect(maxPayload).toBeGreaterThan(500000); }); let kv; const name = `test-${Math.random()}`; it("creates akv, then set and read a large value", async () => { - kv = createAkv({ name }); + kv = createAkv({ name, client }); const val = "z".repeat(maxPayload * 1.5) + "cocalc"; await kv.set("x", val); expect(await kv.get("x")).toBe(val); }); it("cleans up", async () => { - const k = await createDkv({ name }); + const k = await createDkv({ name, client }); k.clear(); await k.close(); }); }); + +afterAll(after); diff --git a/src/packages/backend/conat/test/sync/astream.test.ts b/src/packages/backend/conat/test/sync/astream.test.ts new file mode 100644 index 0000000000..270519930e --- /dev/null +++ b/src/packages/backend/conat/test/sync/astream.test.ts @@ -0,0 +1,232 @@ +/* +Testing basic ops with astream + +DEVELOPMENT: + +pnpm test ./astream.test.ts + +*/ + +import { astream } from "@cocalc/backend/conat/sync"; +import { before, after, connect } from "@cocalc/backend/conat/test/setup"; +import { delay } from "awaiting"; + +beforeAll(before); + +describe("test basics with an astream", () => { + let client, s, s2; + const name = "test-astream"; + + it("creates the astream, then publish and read a value", async () => { + client = connect(); + s = astream({ name, client }); + const { seq } = await s.publish("x"); + expect(seq).toBe(1); + expect(await s.get(1)).toBe("x"); + }); + + it("use a second astream", async () => { + s2 = astream({ name, client, noCache: true }); + expect(await s2.get(1)).toBe("x"); + s2.close(); + }); + + it("publish a message with a header", async () => { + const { seq, time } = await s.publish("has a header", { + headers: { foo: "bar" }, + }); + expect(await s.get(seq)).toBe("has a header"); + expect(await s.headers(seq)).toEqual( + expect.objectContaining({ foo: "bar" }), + ); + // note that seq and time are also in the header + expect(await s.headers(seq)).toEqual({ foo: "bar", seq, time }); + }); + + it("closes, then creates a new astream and sees data is there", async () => { + await s.close(); + s = await astream({ name, client }); + expect(await s.get(1)).toBe("x"); + }); + + it("get full message, which has both the data and the headers", async () => { + const mesg = await s.getMessage(2); + expect(mesg.data).toBe("has a header"); + expect(mesg.headers).toEqual(expect.objectContaining({ foo: "bar" })); + }); + + it("getAll messages", async () => { + const x = await s.getAll(); + const { value } = await x.next(); + expect(value.mesg).toBe("x"); + expect(value.seq).toBe(1); + expect(Math.abs(value.time - Date.now())).toBeLessThan(5000); + const { value: value2 } = await x.next(); + expect(value2.mesg).toBe("has a header"); + expect(value2.headers).toEqual(expect.objectContaining({ foo: "bar" })); + expect(value2.seq).toBe(2); + expect(Math.abs(value2.time - Date.now())).toBeLessThan(5000); + const { done } = await x.next(); + expect(done).toBe(true); + }); + + it("getAll messages starting from the second one", async () => { + const x = await s.getAll({ start_seq: 2, end_seq: 2 }); + const { value } = await x.next(); + expect(value.mesg).toBe("has a header"); + expect(value.seq).toBe(2); + const { done } = await x.next(); + expect(done).toBe(true); + }); + + it("getAll messages starting from the first and ending on the first", async () => { + const x = await s.getAll({ start_seq: 1, end_seq: 1 }); + const { value } = await x.next(); + expect(value.mesg).toBe("x"); + expect(value.seq).toBe(1); + const { done } = await x.next(); + expect(done).toBe(true); + }); + + it("cleans up", () => { + s.close(); + }); +}); + +const stress1 = 1e4; +describe(`stress test -- write, then read back, ${stress1} messages`, () => { + let client, s; + const name = "stress-test"; + + it("creates the astream", async () => { + client = connect(); + s = await astream({ name, client }); + }); + + it(`publishes ${stress1} messages`, async () => { + const v: number[] = []; + for (let i = 0; i < stress1; i++) { + v.push(i); + } + const z = await s.push(...v); + expect(z.length).toBe(stress1); + }); + + it(`reads back ${stress1} messages`, async () => { + const v: any[] = []; + for await (const x of await s.getAll()) { + v.push(x); + } + expect(v.length).toBe(stress1); + }); + + it("cleans up", () => { + s.close(); + }); +}); + +describe("test a changefeed", () => { + let client, s, s2, cf, cf2, cf2b; + const name = "test-astream"; + + it("creates two astreams and three changefeeds on them", async () => { + client = connect(); + s = astream({ name, client }); + cf = await s.changefeed(); + s2 = astream({ name, client, noCache: true }); + cf2 = await s2.changefeed(); + cf2b = await s2.changefeed(); + }); + + it("writes to the stream and sees this in the changefeed", async () => { + const first = cf.next(); + const first2 = cf2.next(); + const first2b = cf2b.next(); + await s.publish("hi"); + + const { value, done } = await first; + expect(done).toBe(false); + + expect(value.mesg).toBe("hi"); + const { value: value2 } = await first2; + expect(value2.mesg).toBe("hi"); + const { value: value2b } = await first2b; + expect(value2b.mesg).toBe("hi"); + }); + + it("verify the three changefeeds are all distinct and do not interfere with each other", async () => { + // write 2 messages and see they are received independently + await s.publish("one"); + await s.publish("two"); + expect((await cf.next()).value.mesg).toBe("one"); + expect((await cf.next()).value.mesg).toBe("two"); + expect((await cf2.next()).value.mesg).toBe("one"); + expect((await cf2b.next()).value.mesg).toBe("one"); + expect((await cf2.next()).value.mesg).toBe("two"); + expect((await cf2b.next()).value.mesg).toBe("two"); + }); + + const stress = 10000; + it(`stress test -- write ${stress} values`, async () => { + const v: number[] = []; + for (let i = 0; i < stress; i++) { + v.push(i); + } + const z = await s.push(...v); + expect(z.length).toBe(v.length); + }); + + it(`stress test getting ${stress} values from a changefeed`, async () => { + for (let i = 0; i < stress; i++) { + await cf.next(); + } + }); + + it("cleans up", () => { + s.close(); + s2.close(); + }); +}); + +describe("test setting with key, ttl and msgID", () => { + let client, s; + const name = "test-astream-sets"; + + it("creates the astream, then publish and read a value", async () => { + client = connect(); + s = astream({ name, client }); + const { seq } = await s.publish("x", { + key: "y", + headers: { with: "key" }, + }); + expect(seq).toBe(1); + expect(await s.get(1)).toBe("x"); + expect(await s.get("y")).toBe("x"); + expect(await s.headers("y")).toEqual( + expect.objectContaining({ with: "key" }), + ); + }); + + it("publish a value with msgID twice and sees that it only appears once", async () => { + const { seq } = await s.publish("foo", { msgID: "xx" }); + const { seq: seq2 } = await s.publish("foo", { msgID: "xx" }); + expect(seq).toEqual(seq2); + }); + + it("publish a value with ttl and sees it vanishes as expected", async () => { + await s.config({ allow_msg_ttl: true }); + const { seq } = await s.publish("foo", { key: "i-have-ttl", ttl: 25 }); + expect(await s.get("i-have-ttl")).toBe("foo"); + await delay(50); + // call config to force enforcing limits + await s.config(); + expect(await s.get("i-have-ttl")).toBe(undefined); + expect(await s.get(seq)).toBe(undefined); + }); + + it("cleans up", () => { + s.close(); + }); +}); + +afterAll(after); diff --git a/src/packages/backend/conat/test/sync/binary.test.ts b/src/packages/backend/conat/test/sync/binary.test.ts new file mode 100644 index 0000000000..bb299dfed7 --- /dev/null +++ b/src/packages/backend/conat/test/sync/binary.test.ts @@ -0,0 +1,99 @@ +/* +Test using binary data with kv and stream. + +You can just store binary directly in kv and stream, since MsgPack +handles buffers just fine. + +DEVELOPMENT: + +pnpm test ./binary.test.ts +*/ + +import { dstream, dkv } from "@cocalc/backend/conat/sync"; +import { before, after, connect } from "@cocalc/backend/conat/test/setup"; +import { wait } from "@cocalc/backend/conat/test/util"; + +beforeAll(before); + +let maxPayload; + +describe("test binary data with a dstream", () => { + let s, + name = `${Math.random()}`; + + // binary values come back as Uint8Array with streams + const data10 = Uint8Array.from(Buffer.from("x".repeat(10))); + it("creates a binary dstream and writes/then reads binary data to/from it", async () => { + s = await dstream({ name }); + expect(s.name).toBe(name); + s.publish(data10); + expect(s.get(0).length).toEqual(data10.length); + await s.save(); + s.close(); + s = await dstream({ name }); + expect(s.get(0).length).toEqual(data10.length); + }); + + it("sanity check on the max payload", async () => { + const client = connect(); + await wait({ until: () => client.info != null }); + maxPayload = client.info?.max_payload ?? 0; + expect(maxPayload).toBeGreaterThan(500000); + }); + + it("writes large binary data to the dstream to test chunking", async () => { + s = await dstream({ name }); + const data = Uint8Array.from(Buffer.from("x".repeat(maxPayload * 1.5))); + s.publish(data); + expect(s.get(s.length - 1).length).toEqual(data.length); + await s.save(); + s.close(); + s = await dstream({ name }); + expect(s.get(s.length - 1).length).toEqual(data.length); + }); + + it("clean up", async () => { + await s.delete({ all: true }); + await s.close(); + }); +}); + +describe("test binary data with a dkv", () => { + let s, + name = `${Math.random()}`; + + // binary values come back as buffer with dkv + const data10 = Buffer.from("x".repeat(10)); + + it("creates a binary dkv and writes/then reads binary data to/from it", async () => { + s = await dkv({ name }); + expect(s.name).toBe(name); + s.x = data10; + expect(s.x).toEqual(data10); + expect(s.x.length).toEqual(data10.length); + await s.save(); + s.close(); + s = await dkv({ name }); + await wait({ until: () => s.has("x") }); + expect(s.x.length).toEqual(data10.length); + expect(s.x).toEqual(data10); + }); + + it("writes large binary data to the dkv to test chunking", async () => { + s = await dkv({ name }); + const data = Uint8Array.from(Buffer.from("x".repeat(maxPayload * 1.5))); + s.y = data; + expect(s.y.length).toEqual(data.length); + await s.save(); + s.close(); + s = await dkv({ name }); + expect(s.y.length).toEqual(data.length); + }); + + it("clean up", async () => { + await s.clear(); + s.close(); + }); +}); + +afterAll(after); diff --git a/src/packages/backend/conat/test/sync/connectivity.test.ts b/src/packages/backend/conat/test/sync/connectivity.test.ts new file mode 100644 index 0000000000..4a4f61e47e --- /dev/null +++ b/src/packages/backend/conat/test/sync/connectivity.test.ts @@ -0,0 +1,71 @@ +/* +Tests that various sync functionality works after restarting the conat server. + +pnpm test ./connectivity.test.ts + +*/ + +import { dkv } from "@cocalc/backend/conat/sync"; +import { + before, + after, + restartServer, + restartPersistServer, + setDefaultTimeouts, +} from "@cocalc/backend/conat/test/setup"; + +beforeAll(before); + +describe("test that dkv survives server restart", () => { + let kv; + const name = `test-${Math.random()}`; + + it("restarts the conat socketio server to make sure that works", async () => { + // some tests below will randomly sometimes take longer than 5s without this: + setDefaultTimeouts({ request: 250, publish: 250 }); + await restartServer(); + }); + + it("creates the dkv and does a basic test", async () => { + kv = await dkv({ name }); + kv.a = 10; + expect(kv.a).toEqual(10); + await kv.save(); + expect(kv.hasUnsavedChanges()).toBe(false); + }); + + it("restart the socketio server and confirm that dkv still works", async () => { + await restartServer(); + kv.b = 7; + expect(kv.b).toEqual(7); + await kv.save(); + expect(kv.hasUnsavedChanges()).toBe(false); + }); + + it("restart again (without await) the socketio server and confirm that dkv still works", async () => { + restartServer(); + kv.b = 77; + expect(kv.b).toEqual(77); + await kv.save(); + expect(kv.hasUnsavedChanges()).toBe(false); + }); + + it("restart persist server", async () => { + await restartPersistServer(); + kv.b = 123; + expect(kv.b).toEqual(123); + await kv.save(); + expect(kv.hasUnsavedChanges()).toBe(false); + }); + + jest.setTimeout(10000); + it("restart both servers at once", async () => { + await Promise.all([restartPersistServer(), restartServer()]); + kv.b = 389; + expect(kv.b).toEqual(389); + await kv.save(); + expect(kv.hasUnsavedChanges()).toBe(false); + }); +}); + +afterAll(after); diff --git a/src/packages/backend/nats/test/sync/dko.test.ts b/src/packages/backend/conat/test/sync/dko.test.ts similarity index 77% rename from src/packages/backend/nats/test/sync/dko.test.ts rename to src/packages/backend/conat/test/sync/dko.test.ts index 562e5e82eb..fc29092cdc 100644 --- a/src/packages/backend/nats/test/sync/dko.test.ts +++ b/src/packages/backend/conat/test/sync/dko.test.ts @@ -3,12 +3,15 @@ Testing basic ops with dko = distributed key:object store with SPARSE updates. DEVELOPMENT: -pnpm exec jest --forceExit "dko.test.ts" +pnpm test ./dko.test.ts */ -import { dko as createDko } from "@cocalc/backend/nats/sync"; -import { getMaxPayload } from "@cocalc/nats/util"; +import { dko as createDko } from "@cocalc/backend/conat/sync"; +import { before, after, connect } from "@cocalc/backend/conat/test/setup"; +import { wait } from "@cocalc/backend/conat/test/util"; + +beforeAll(before); describe("create a public dko and do a basic operation", () => { let kv; @@ -72,12 +75,20 @@ describe("test a large value that requires chunking", () => { let kv; const name = `test-${Math.random()}`; + let maxPayload = 0; + + it("sanity check on the max payload", async () => { + const client = connect(); + await wait({ until: () => client.info != null }); + maxPayload = client.info?.max_payload ?? 0; + expect(maxPayload).toBeGreaterThan(500000); + }); + it("creates the dko", async () => { kv = await createDko({ name }); expect(kv.getAll()).toEqual({}); - const n = await getMaxPayload(); - const big = { foo: "b".repeat(n * 1.3) }; + const big = { foo: "b".repeat(maxPayload * 1.3) }; kv.set("big", big); expect(kv.get("big")).toEqual(big); }); @@ -87,3 +98,5 @@ describe("test a large value that requires chunking", () => { await kv.close(); }); }); + +afterAll(after); diff --git a/src/packages/backend/conat/test/sync/dkv-basics.test.ts b/src/packages/backend/conat/test/sync/dkv-basics.test.ts new file mode 100644 index 0000000000..9408dc133a --- /dev/null +++ b/src/packages/backend/conat/test/sync/dkv-basics.test.ts @@ -0,0 +1,110 @@ +/* +DEVELOPMENT: + +pnpm test ./dkv-basics.test.ts + +*/ +import { DKV } from "@cocalc/conat/sync/dkv"; +import { connect, before, after } from "@cocalc/backend/conat/test/setup"; +import { wait } from "@cocalc/backend/conat/test/util"; + +beforeAll(before); + +describe("create a general kv and do basic operations", () => { + const name = "test"; + let client, kv; + + it("creates the kv", async () => { + client = connect(); + kv = new DKV({ name, client }); + await kv.init(); + }); + + it("sets and deletes a key", async () => { + expect(kv.has("foo")).toBe(false); + kv.set("foo", 10); + expect(kv.has("foo")).toBe(true); + expect(kv.getAll()).toEqual({ foo: 10 }); + kv.delete("foo"); + expect(kv.getAll()).toEqual({}); + kv.set("co", "nat"); + await kv.save(); + }); + + let client2, kv2; + it("view the kv from a second client via sync, set a date value and observe it syncs", async () => { + client2 = connect(); + kv2 = new DKV({ name, client: client2 }); + await kv2.init(); + expect(kv2.getAll()).toEqual({ co: "nat" }); + + const date = new Date("1974"); + kv2.set("x", date); + // replication is not instant + expect(kv.get("x")).toBe(undefined); + await kv2.save(); + await wait({ until: () => kv.get("x") }); + expect(kv.getAll()).toEqual({ x: date, co: "nat" }); + }); + + it("checks that clear works", async () => { + kv.clear(); + await wait({ until: () => kv.length == 0 }); + expect(kv.length).toBe(0); + await wait({ until: () => kv2.length == 0 }); + }); + + it("checks that time works", async () => { + const key = "x".repeat(10000); + kv.set(key, "big key"); + await kv.save(); + expect(Math.abs(Date.now() - kv.time(key))).toBeLessThan(300); + expect(kv.time()).toEqual({ [key]: kv.time(key) }); + expect(kv2.time()).toEqual({ [key]: kv2.time(key) }); + }); + + it("check headers work", async () => { + kv.set("big", "headers", { headers: { silicon: "valley", x: { y: "z" } } }); + // this uses local state + expect(kv.headers("big")).toEqual({ silicon: "valley", x: { y: "z" } }); + await kv.save(); + // this uses what got echoed back from server + expect(kv.headers("big")).toEqual({ silicon: "valley", x: { y: "z" } }); + expect(kv2.headers("big")).toEqual({ silicon: "valley", x: { y: "z" } }); + }); + + it("checks hasUnsavedChanges works", async () => { + expect(kv.hasUnsavedChanges()).toBe(false); + kv.set("unsaved", ["changes"]); + expect(kv.hasUnsavedChanges()).toBe(true); + expect(kv.unsavedChanges()).toEqual(["unsaved"]); + expect(kv2.hasUnsavedChanges()).toBe(false); + await kv.save(); + expect(kv.hasUnsavedChanges()).toBe(false); + }); + + it("checks stats works", () => { + const { bytes, count } = kv.stats(); + expect(bytes).not.toBeNaN(); + expect(bytes).toBeGreaterThan(0); + expect(count).not.toBeNaN(); + expect(count).toBeGreaterThan(0); + }); + + it("checks seq is ", async () => { + kv.set("x", "11"); + await kv.save(); + const seq = kv.seq("x"); + expect(seq).toBeGreaterThan(0); + kv.set("x", 15); + await kv.save(); + expect(kv.seq("x") - seq).toBe(1); + }); + + it("clean up", async () => { + kv.close(); + client.close(); + }); +}); + +afterAll(after); diff --git a/src/packages/backend/nats/test/sync/dkv-merge.test.ts b/src/packages/backend/conat/test/sync/dkv-merge.test.ts similarity index 92% rename from src/packages/backend/nats/test/sync/dkv-merge.test.ts rename to src/packages/backend/conat/test/sync/dkv-merge.test.ts index c80a73c044..e78c6e876a 100644 --- a/src/packages/backend/nats/test/sync/dkv-merge.test.ts +++ b/src/packages/backend/conat/test/sync/dkv-merge.test.ts @@ -3,16 +3,19 @@ Testing merge conflicts with dkv DEVELOPMENT: -pnpm exec jest --watch --forceExit "dkv-merge.test.ts" +pnpm test ./dkv-merge.test.ts */ -import { dkv as createDkv } from "@cocalc/backend/nats/sync"; +import { dkv as createDkv } from "@cocalc/backend/conat/sync"; import { once } from "@cocalc/util/async-utils"; import { diff_match_patch } from "@cocalc/util/dmp"; +import { before, after } from "@cocalc/backend/conat/test/setup"; + +beforeAll(before); async function getKvs(opts?) { - const name = `test-${Math.random()}`; + const name = `test${Math.round(1000 * Math.random())}`; // We disable autosave so that we have more precise control of how conflicts // get resolved, etc. for testing purposes. const kv1 = await createDkv({ @@ -27,6 +30,10 @@ async function getKvs(opts?) { ...opts, noCache: true, }); + // @ts-ignore -- a little double check + if (kv1.kv === kv2.kv) { + throw Error("must not being using same underlying kv"); + } return { kv1, kv2 }; } @@ -181,3 +188,5 @@ describe("test a 3-way merge of that merges objects", () => { expect(kv2.get("x")).toEqual({ a: 5, b: 15, c: 12, d: 3 }); }); }); + +afterAll(after); diff --git a/src/packages/backend/nats/test/sync/dkv.test.ts b/src/packages/backend/conat/test/sync/dkv.test.ts similarity index 75% rename from src/packages/backend/nats/test/sync/dkv.test.ts rename to src/packages/backend/conat/test/sync/dkv.test.ts index 3c246e9f6f..6082e18e09 100644 --- a/src/packages/backend/nats/test/sync/dkv.test.ts +++ b/src/packages/backend/conat/test/sync/dkv.test.ts @@ -3,20 +3,24 @@ Testing basic ops with dkv DEVELOPMENT: -pnpm test dkv.test.ts +pnpm test ./dkv.test.ts */ -import { dkv as createDkv } from "@cocalc/backend/nats/sync"; +import { dkv as createDkv } from "@cocalc/backend/conat/sync"; import { once } from "@cocalc/util/async-utils"; import { delay } from "awaiting"; +import { before, after, connect } from "@cocalc/backend/conat/test/setup"; +import { wait } from "@cocalc/backend/conat/test/util"; + +beforeAll(before); describe("create a public dkv and do basic operations", () => { let kv; const name = `test-${Math.random()}`; it("creates the dkv", async () => { - kv = await createDkv({ name }); + kv = await createDkv({ name, noCache: true }); expect(kv.getAll()).toEqual({}); }); @@ -28,14 +32,14 @@ describe("create a public dkv and do basic operations", () => { it("waits for the dkv to be longterm saved, then closing and recreates the kv and verifies that the key is there.", async () => { await kv.save(); kv.close(); - kv = await createDkv({ name }); + kv = await createDkv({ name, noCache: true }); expect(kv.a).toEqual(10); }); it("closes the kv", async () => { await kv.clear(); - kv.close(); - expect(kv.getAll).toThrow("closed"); + await kv.close(); + expect(() => kv.getAll()).toThrow("closed"); }); }); @@ -52,14 +56,15 @@ describe("opens a dkv twice and verifies the cache works and is reference counte }); it("closes kv1 (one reference)", async () => { - kv1.close(); + await kv1.close(); expect(kv2.getAll).not.toThrow(); }); it("closes kv2 (another reference)", async () => { - kv2.close(); + await kv2.close(); + await delay(1); // really closed! - expect(kv2.getAll).toThrow("closed"); + expect(() => kv2.getAll()).toThrow("closed"); }); it("create and see it is new now", async () => { @@ -244,18 +249,19 @@ describe("set several items, confirm write worked, save, and confirm they are st expect(Object.keys(kv.getAll()).length).toEqual(count); expect(kv.getAll()).toEqual(obj); await kv.save(); - expect(Date.now() - t0).toBeLessThan(1000); + expect(Date.now() - t0).toBeLessThan(2000); + await wait({ until: () => Object.keys(kv.getAll()).length == count }); expect(Object.keys(kv.getAll()).length).toEqual(count); // // the local state maps should also get cleared quickly, // // but there is no event for this, so we loop: // @ts-ignore: saved is private - while (Object.keys(kv.generalDKV.local).length > 0) { + while (Object.keys(kv.local).length > 0) { await delay(10); } // @ts-ignore: local is private - expect(kv.generalDKV.local).toEqual({}); + expect(kv.local).toEqual({}); // @ts-ignore: saved is private - expect(kv.generalDKV.saved).toEqual({}); + expect(kv.saved).toEqual({}); await kv.clear(); await kv.close(); @@ -264,7 +270,7 @@ describe("set several items, confirm write worked, save, and confirm they are st describe("do an insert and clear test", () => { const name = `test-${Math.random()}`; - const count = 100; + const count = 25; it(`adds ${count} entries, saves, clears, and confirms empty`, async () => { const kv = await createDkv({ name }); expect(kv.getAll()).toEqual({}); @@ -273,10 +279,12 @@ describe("do an insert and clear test", () => { } expect(Object.keys(kv.getAll()).length).toEqual(count); await kv.save(); + await wait({ until: () => Object.keys(kv.getAll()).length == count }); expect(Object.keys(kv.getAll()).length).toEqual(count); kv.clear(); expect(kv.getAll()).toEqual({}); await kv.save(); + await wait({ until: () => Object.keys(kv.getAll()).length == 0 }); expect(kv.getAll()).toEqual({}); }); }); @@ -333,10 +341,10 @@ describe("create many distinct clients at once, write to all of them, and see th }); }); -describe("tests involving null/undefined values", () => { +describe("tests involving null/undefined values and delete", () => { let kv1; let kv2; - const name = `test-${Math.random()}`; + const name = `test-${Math.round(100 * Math.random())}`; it("creates the dkv twice", async () => { kv1 = await createDkv({ name, noAutosave: true, noCache: true }); @@ -345,16 +353,17 @@ describe("tests involving null/undefined values", () => { expect(kv1 === kv2).toBe(false); }); - it("sets a value to null, which is fully supported like any other value", () => { - kv1.a = null; - expect(kv1.a).toBe(null); - expect(kv1.a === null).toBe(true); - expect(kv1.length).toBe(1); - }); + // it("sets a value to null, which is fully supported like any other value", () => { + // kv1.a = null; + // expect(kv1.a).toBe(null); + // expect(kv1.a === null).toBe(true); + // expect(kv1.length).toBe(1); + // }); it("make sure null value sync's as expected", async () => { - kv1.save(); - await once(kv2, "change"); + kv1.a = null; + await kv1.save(); + await wait({ until: () => kv2.has("a") }); expect(kv2.a).toBe(null); expect(kv2.a === null).toBe(true); expect(kv2.length).toBe(1); @@ -362,6 +371,7 @@ describe("tests involving null/undefined values", () => { it("sets a value to undefined, which is the same as deleting a value", () => { kv1.a = undefined; + expect(kv1.has("a")).toBe(false); expect(kv1.a).toBe(undefined); expect(kv1.a === undefined).toBe(true); expect(kv1.length).toBe(0); @@ -369,8 +379,8 @@ describe("tests involving null/undefined values", () => { }); it("make sure undefined (i.e., delete) sync's as expected", async () => { - kv1.save(); - await once(kv2, "change"); + await kv1.save(); + await wait({ until: () => kv2.a === undefined }); expect(kv2.a).toBe(undefined); expect(kv2.a === undefined).toBe(true); expect(kv2.length).toBe(0); @@ -395,72 +405,90 @@ describe("tests involving null/undefined values", () => { }); }); -import { numSubscriptions } from "@cocalc/nats/client"; +describe("ensure there isn't a really obvious subscription leak", () => { + let client; -describe("ensure there are no NATS subscription leaks", () => { - // There is some slight slack at some point due to the clock stuff, - // inventory, etc. It is constant and small, whereas we allocate - // a large number of kv's in the test. - const SLACK = 4; + it("create a client, which initially has only one subscription (the inbox)", async () => { + client = connect(); + await client.getInbox(); + expect(client.numSubscriptions()).toBe(1); + }); - it("creates and closes many kv's and checks there is no leak", async () => { - const before = numSubscriptions(); - const COUNT = 20; + const count = 10; + it(`creates and closes ${count} streams and checks there is no leak`, async () => { + const before = client.numSubscriptions(); // create const a: any = []; - for (let i = 0; i < COUNT; i++) { + for (let i = 0; i < count; i++) { a[i] = await createDkv({ name: `${Math.random()}`, noAutosave: true, noCache: true, }); } - for (let i = 0; i < COUNT; i++) { + // NOTE: in fact there's very unlikely to be a subscription leak, since + // dkv's don't use new subscriptions -- they all use requestMany instead + // to a common inbox prefix, and there's just one subscription for an inbox. + expect(client.numSubscriptions()).toEqual(before); + for (let i = 0; i < count; i++) { await a[i].close(); } - const after = numSubscriptions(); - expect(Math.abs(after - before)).toBeLessThan(SLACK); + const after = client.numSubscriptions(); + expect(after).toBe(before); + + // also check count on server went down. + expect((await client.getSubscriptions()).size).toBe(before); }); it("does another leak test, but with a set operation each time", async () => { - const before = numSubscriptions(); - const COUNT = 20; + const before = client.numSubscriptions(); // create const a: any = []; - for (let i = 0; i < COUNT; i++) { + for (let i = 0; i < count; i++) { a[i] = await createDkv({ name: `${Math.random()}`, noAutosave: true, noCache: true, }); - a[i].set(i, i); + a[i].set(`${i}`, i); await a[i].save(); } - for (let i = 0; i < COUNT; i++) { - a[i].clear(); + // this isn't going to be a problem: + expect(client.numSubscriptions()).toEqual(before); + for (let i = 0; i < count; i++) { await a[i].close(); } - const after = numSubscriptions(); - expect(Math.abs(after - before)).toBeLessThan(SLACK); + const after = client.numSubscriptions(); + expect(after).toBe(before); }); +}); - it("does another leak test, but opening and immediately closing and doing a set operation each time", async () => { - const before = numSubscriptions(); - const COUNT = 20; - // create - const a: any = []; - for (let i = 0; i < COUNT; i++) { - a[i] = await createDkv({ - name: `${Math.random()}`, - noAutosave: true, - noCache: true, - }); - a[i].set(i, i); - await a[i].save(); - a[i].clear(); - await a[i].close(); - } - const after = numSubscriptions(); - expect(Math.abs(after - before)).toBeLessThan(SLACK); +describe("test creating and closing a dkv doesn't leak subscriptions", () => { + let client; + let kv; + let name = "sub.count"; + let subs; + + it("make a new client and count subscriptions", async () => { + client = connect(); + await once(client, "connected"); + await client.getInbox(); + subs = client.numSubscriptions(); + expect(subs).toBe(1); // the inbox + }); + + it("creates dkv", async () => { + kv = await createDkv({ name }); + kv.set("x", 5); + await wait({ until: () => kv.length == 1 }); + }); + + it("close the kv and confirm subs returns to 1", async () => { + kv.close(); + await expect(() => { + client.numSubscriptions() == 1; + }); }); }); + +afterAll(after); diff --git a/src/packages/backend/nats/test/sync/estream.test.ts b/src/packages/backend/conat/test/sync/dstream-ephemeral.test.ts similarity index 67% rename from src/packages/backend/nats/test/sync/estream.test.ts rename to src/packages/backend/conat/test/sync/dstream-ephemeral.test.ts index 6d5104f29d..89dbbc3ea2 100644 --- a/src/packages/backend/nats/test/sync/estream.test.ts +++ b/src/packages/backend/conat/test/sync/dstream-ephemeral.test.ts @@ -2,22 +2,32 @@ Testing basic ops with dsteam (distributed streams), but all are ephemeral. The first tests are initially similar to those for dstream.test.ts, but with -{ephemeral: true, leader:true}. There are also further tests of the client/server aspects. +{ephemeral: true}. There are also further tests of the client/server aspects. DEVELOPMENT: -pnpm test estream.test.ts +pnpm test ./dstream-ephemeral.test.ts */ +import { connect, before, after, wait } from "@cocalc/backend/conat/test/setup"; import { createDstreamEphemeral as create } from "./util"; -import { dstream as createDstream0 } from "@cocalc/backend/nats/sync"; -import { once } from "@cocalc/util/async-utils"; +import { dstream as createDstream0 } from "@cocalc/backend/conat/sync"; +//import { delay } from "awaiting"; + +beforeAll(before); async function createDstream(opts) { - return await createDstream0({ ephemeral: true, leader: true, ...opts }); + return await createDstream0({ + noCache: true, + noAutosave: true, + ephemeral: true, + ...opts, + }); } +jest.setTimeout(10000); + describe("create a dstream and do some basic operations", () => { let s; @@ -53,6 +63,8 @@ describe("create a dstream and do some basic operations", () => { await s.close(); // using s fails expect(s.getAll).toThrow("closed"); + // wait for server to discard stream data + // (it's instant right now!) // create new stream with same name const t = await createDstream({ name }); // ensure it is NOT just from the cache @@ -65,14 +77,11 @@ describe("create a dstream and do some basic operations", () => { describe("create two dstreams and observe sync between them", () => { const name = `test-${Math.random()}`; let s1, s2; + let client2; it("creates two distinct dstream objects s1 and s2 with the same name", async () => { - s1 = await createDstream({ name, noAutosave: true, noCache: true }); - s2 = await createDstream({ - name, - noAutosave: true, - noCache: true, - leader: false, - }); + client2 = connect(); + s1 = await createDstream({ name }); + s2 = await createDstream({ client: client2, name }); // definitely distinct expect(s1 === s2).toBe(false); }); @@ -82,9 +91,7 @@ describe("create two dstreams and observe sync between them", () => { expect(s1[0]).toEqual("hello"); expect(s2.length).toEqual(0); await s1.save(); - while (s2[0] != "hello") { - await once(s2, "change"); - } + await wait({ until: () => s2[0] == "hello" }); expect(s2[0]).toEqual("hello"); expect(s2.getAll()).toEqual(["hello"]); }); @@ -92,10 +99,9 @@ describe("create two dstreams and observe sync between them", () => { it("now write to s2 and save and see that reflected in s1", async () => { s2.push("hi from s2"); await s2.save(); - while (s1[1] != "hi from s2") { - await once(s1, "change"); - } + await wait({ until: () => s1[1] == "hi from s2" && s2[1] == "hi from s2" }); expect(s1[1]).toEqual("hi from s2"); + expect(s2[1]).toEqual("hi from s2"); }); it("s1.stream and s2.stream should be the same right now", () => { @@ -108,21 +114,40 @@ describe("create two dstreams and observe sync between them", () => { expect(s2.getAll()).toEqual(["hello", "hi from s2"]); }); - it("write to s1 and s2 and save at the same time and see some 'random choice' of order gets imposed by the server", async () => { + it("cleans up", () => { + s1.close(); + s2.close(); + client2.close(); + }); +}); + +describe("create two dstreams and test sync with parallel save", () => { + const name = `test-${Math.random()}`; + let s1, s2; + let client2; + it("creates two distinct dstream objects s1 and s2 with the same name", async () => { + client2 = connect(); + s1 = await createDstream({ name }); + s2 = await createDstream({ client: client2, name }); + // definitely distinct + expect(s1 === s2).toBe(false); + }); + + it("write to s1 and s2 and save at the same time", async () => { s1.push("s1"); s2.push("s2"); // our changes are reflected locally - expect(s1.getAll()).toEqual(["hello", "hi from s2", "s1"]); - expect(s2.getAll()).toEqual(["hello", "hi from s2", "s2"]); + expect(s1.getAll()).toEqual(["s1"]); + expect(s2.getAll()).toEqual(["s2"]); // now kick off the two saves *in parallel* s1.save(); s2.save(); - await once(s1, "change"); - while (s2.length != s1.length) { - await once(s2, "change"); - } + await wait({ until: () => s1.length >= 2 && s2.length >= 2 }); expect(s1.getAll()).toEqual(s2.getAll()); - expect(s1.getAll()).toEqual(["hello", "hi from s2", "s1", "s2"]); + }); + + it("cleans up", () => { + client2.close(); }); }); @@ -146,7 +171,7 @@ describe("get sequence number and time of message", () => { it("save and get server assigned sequence number", async () => { s.save(); - await once(s, "change"); + await wait({ until: () => s.seq(0) > 0 }); const n = s.seq(0); expect(n).toBeGreaterThan(0); }); @@ -166,10 +191,8 @@ describe("get sequence number and time of message", () => { }); it("and time is bigger", async () => { - if (s.time(1) == null) { - await once(s, "change"); - } - expect(s.time(0).getTime()).toBeLessThan(s.time(1).getTime()); + await wait({ until: () => s.time(1) != null }); + expect(s.time(0).getTime()).toBeLessThanOrEqual(s.time(1).getTime()); }); }); @@ -182,10 +205,7 @@ describe("testing start_seq", () => { expect(s.getAll()).toEqual([1, 2, 3]); // save, thus getting sequence numbers s.save(); - while (s.seq(2) == null) { - s.save(); - await once(s, "change"); - } + await wait({ until: () => s.seq(2) != null }); seq = [s.seq(0), s.seq(1), s.seq(2)]; // tests partly that these are integers... const n = seq.reduce((a, b) => a + b, 0); @@ -195,10 +215,11 @@ describe("testing start_seq", () => { let t; it("it opens another copy of the stream, but starting with the last sequence number, so only one message", async () => { + const client = connect(); t = await createDstream({ + client, name, noAutosave: true, - leader: false, start_seq: seq[2], }); expect(t.length).toBe(1); @@ -227,8 +248,9 @@ describe("a little bit of a stress test", () => { s.push({ i }); } expect(s.length).toBe(count); - // NOTE: warning -- this is **MUCH SLOWER**, e.g., 10x slower, - // running under jest, hence why count is small. + // [ ] TODO rewrite this save to send everything in a single message + // which gets chunked, will we be much faster, then change the count + // above to 1000 or 10000. await s.save(); expect(s.length).toBe(count); }); @@ -249,38 +271,41 @@ describe("dstream typescript test", () => { }); }); -import { numSubscriptions } from "@cocalc/nats/client"; +describe("ensure there isn't a really obvious subscription leak", () => { + let client; -describe("ensure there are no NATS subscription leaks", () => { - // There is some slight slack at some point due to the clock stuff, - // inventory, etc. It is constant and small, whereas we allocate - // a large number of kv's in the test. - const SLACK = 4; + it("create a client, which initially has only one subscription (the inbox)", async () => { + client = connect(); + expect(client.numSubscriptions()).toBe(0); + await client.getInbox(); + expect(client.numSubscriptions()).toBe(1); + }); - it("creates and closes many kv's and checks there is no leak", async () => { - const before = numSubscriptions(); - const COUNT = 20; + const count = 100; + it(`creates and closes ${count} streams and checks there is no leak`, async () => { + const before = client.numSubscriptions(); // create const a: any = []; - for (let i = 0; i < COUNT; i++) { + for (let i = 0; i < count; i++) { a[i] = await createDstream({ name: `${Math.random()}`, - noAutosave: true, }); } - for (let i = 0; i < COUNT; i++) { + for (let i = 0; i < count; i++) { await a[i].close(); } - const after = numSubscriptions(); - expect(Math.abs(after - before)).toBeLessThan(SLACK); + const after = client.numSubscriptions(); + expect(after).toBe(before); + + // also check count on server went down. + expect((await client.getSubscriptions()).size).toBe(before); }); - it("does another leak test, but with a set operation each time", async () => { - const before = numSubscriptions(); - const COUNT = 20; + it("does another leak test, but with a publish operation each time", async () => { + const before = client.numSubscriptions(); // create const a: any = []; - for (let i = 0; i < COUNT; i++) { + for (let i = 0; i < count; i++) { a[i] = await createDstream({ name: `${Math.random()}`, noAutosave: true, @@ -288,11 +313,12 @@ describe("ensure there are no NATS subscription leaks", () => { a[i].publish(i); await a[i].save(); } - for (let i = 0; i < COUNT; i++) { - await a[i].purge(); + for (let i = 0; i < count; i++) { await a[i].close(); } - const after = numSubscriptions(); - expect(Math.abs(after - before)).toBeLessThan(SLACK); + const after = client.numSubscriptions(); + expect(after).toBe(before); }); }); + +afterAll(after); diff --git a/src/packages/backend/nats/test/sync/dstream.test.ts b/src/packages/backend/conat/test/sync/dstream.test.ts similarity index 62% rename from src/packages/backend/nats/test/sync/dstream.test.ts rename to src/packages/backend/conat/test/sync/dstream.test.ts index 2154b98d48..f5180750b7 100644 --- a/src/packages/backend/nats/test/sync/dstream.test.ts +++ b/src/packages/backend/conat/test/sync/dstream.test.ts @@ -1,15 +1,20 @@ /* -Testing basic ops with dsteam (distributed streams) +Testing basic ops with *persistent* dstreams. DEVELOPMENT: -pnpm test dstream.test.ts +pnpm test ./dstream.test.ts */ import { createDstream as create } from "./util"; -import { dstream as createDstream } from "@cocalc/backend/nats/sync"; +import { dstream as createDstream } from "@cocalc/backend/conat/sync"; import { once } from "@cocalc/util/async-utils"; +import { connect, before, after, wait } from "@cocalc/backend/conat/test/setup"; + +beforeAll(before); + +jest.setTimeout(10000); describe("create a dstream and do some basic operations", () => { let s; @@ -93,13 +98,15 @@ describe("create two dstreams and observe sync between them", () => { // now kick off the two saves *in parallel* s1.save(); s2.save(); - await once(s1, "change"); - if (s2.length != s1.length) { - await once(s2, "change"); - } + await wait({ + until: () => { + return s1.length == 4 && s2.length == 4; + }, + }); expect(s1.getAll()).toEqual(s2.getAll()); - // in fact s1,s2 is the order since we called s1.save first: - expect(s1.getAll()).toEqual(["hello", "hi from s2", "s1", "s2"]); + expect(new Set(s1.getAll())).toEqual( + new Set(["hello", "hi from s2", "s1", "s2"]), + ); }); }); @@ -155,7 +162,8 @@ describe("closing also saves by default, but not if autosave is off", () => { const name = `test-${Math.random()}`; it("creates stream and write a message", async () => { - s = await createDstream({ name, noAutosave: false /* the default */ }); + // noAutosave: false is the default: + s = await createDstream({ name, noAutosave: false }); s.push(389); }); @@ -215,6 +223,36 @@ describe("testing start_seq", () => { expect(s.getAll()).toEqual([2, 3]); expect(s.start_seq).toEqual(seq[1]); }); + + it("a bigger example involving loading older messages", async () => { + for (let i = 4; i < 100; i++) { + s.push(i); + } + await s.save(); + const last = s.seq(s.length - 1); + const mid = s.seq(s.length - 50); + await s.close(); + s = await createDstream({ + name, + noAutosave: true, + start_seq: last, + }); + expect(s.length).toBe(1); + expect(s.getAll()).toEqual([99]); + expect(s.start_seq).toEqual(last); + + await s.load({ start_seq: mid }); + expect(s.length).toEqual(50); + expect(s.start_seq).toEqual(mid); + for (let i = 0; i < 50; i++) { + expect(s.get(i)).toBe(i + 50); + } + + await s.load({ start_seq: 0 }); + for (let i = 0; i < 99; i++) { + expect(s.get(i)).toBe(i + 1); + } + }); }); describe("a little bit of a stress test", () => { @@ -252,38 +290,40 @@ describe("dstream typescript test", () => { }); }); -import { numSubscriptions } from "@cocalc/nats/client"; +describe("ensure there isn't a really obvious subscription leak", () => { + let client; -describe("ensure there are no NATS subscription leaks", () => { - // There is some slight slack at some point due to the clock stuff, - // inventory, etc. It is constant and small, whereas we allocate - // a large number of kv's in the test. - const SLACK = 4; + it("create a client, which initially has only one subscription (the inbox)", async () => { + client = connect(); + await client.getInbox(); + expect(client.numSubscriptions()).toBe(1); + }); - it("creates and closes many kv's and checks there is no leak", async () => { - const before = numSubscriptions(); - const COUNT = 20; + const count = 100; + it(`creates and closes ${count} streams and checks there is no leak`, async () => { + const before = client.numSubscriptions(); // create const a: any = []; - for (let i = 0; i < COUNT; i++) { + for (let i = 0; i < count; i++) { a[i] = await createDstream({ name: `${Math.random()}`, - noAutosave: true, }); } - for (let i = 0; i < COUNT; i++) { + for (let i = 0; i < count; i++) { await a[i].close(); } - const after = numSubscriptions(); - expect(Math.abs(after - before)).toBeLessThan(SLACK); + const after = client.numSubscriptions(); + expect(after).toBe(before); + + // also check count on server went down. + expect((await client.getSubscriptions()).size).toBe(before); }); - it("does another leak test, but with a set operation each time", async () => { - const before = numSubscriptions(); - const COUNT = 20; + it("does another leak test, but with a publish operation each time", async () => { + const before = client.numSubscriptions(); // create const a: any = []; - for (let i = 0; i < COUNT; i++) { + for (let i = 0; i < count; i++) { a[i] = await createDstream({ name: `${Math.random()}`, noAutosave: true, @@ -291,11 +331,83 @@ describe("ensure there are no NATS subscription leaks", () => { a[i].publish(i); await a[i].save(); } - for (let i = 0; i < COUNT; i++) { - await a[i].purge(); + for (let i = 0; i < count; i++) { await a[i].close(); } - const after = numSubscriptions(); - expect(Math.abs(after - before)).toBeLessThan(SLACK); + const after = client.numSubscriptions(); + expect(after).toBe(before); }); }); + +describe("test delete of messages from stream", () => { + let client1, client2, s1, s2; + const name = "test-delete"; + it("create two clients", async () => { + client1 = connect(); + client2 = connect(); + s1 = await createDstream({ + client: client1, + name, + noAutosave: true, + noCache: true, + }); + s2 = await createDstream({ + client: client2, + name, + noAutosave: true, + noCache: true, + }); + }); + + it("writes message one, confirm seen by other, then delete and confirm works", async () => { + s1.push("hello"); + await s1.save(); + await wait({ until: () => s2.length > 0 }); + s1.delete({ all: true }); + await wait({ until: () => s2.length == 0 && s1.length == 0 }); + }); + + it("same delete test as above but with a few more items and delete on s2 instead", async () => { + for (let i = 0; i < 10; i++) { + s1.push(i); + } + await s1.save(); + await wait({ until: () => s2.length == 10 }); + s2.delete({ all: true }); + await wait({ until: () => s2.length == 0 && s1.length == 0 }); + }); + + it("delete specific index", async () => { + s1.push("x", "y", "z"); + await s1.save(); + await wait({ until: () => s2.length == 3 }); + s2.delete({ last_index: 1 }); + await wait({ until: () => s2.length == 1 && s1.length == 1 }); + expect(s1.get()).toEqual(["z"]); + }); + + it("delete specific seq number", async () => { + s1.push("x", "y"); + await s1.save(); + expect(s1.get()).toEqual(["z", "x", "y"]); + const seq = s1.seq(1); + const { seqs } = await s1.delete({ seq }); + expect(seqs).toEqual([seq]); + await wait({ until: () => s2.length == 2 && s1.length == 2 }); + expect(s1.get()).toEqual(["z", "y"]); + }); + + it("delete up to a sequence number", async () => { + s1.push("x", "y"); + await s1.save(); + expect(s1.get()).toEqual(["z", "y", "x", "y"]); + const seq = s1.seq(1); + const { seqs } = await s1.delete({ last_seq: seq }); + expect(seqs.length).toBe(2); + expect(seqs[1]).toBe(seq); + await wait({ until: () => s1.length == 2 }); + expect(s1.get()).toEqual(["x", "y"]); + }); +}); + +afterAll(after); diff --git a/src/packages/backend/conat/test/sync/headers.test.ts b/src/packages/backend/conat/test/sync/headers.test.ts new file mode 100644 index 0000000000..78bc29a45c --- /dev/null +++ b/src/packages/backend/conat/test/sync/headers.test.ts @@ -0,0 +1,80 @@ +/* +Test using user-defined headers with kv and stream. + +DEVELOPMENT: + +pnpm test ./headers.test.ts +*/ + +import { dstream, dkv } from "@cocalc/backend/conat/sync"; +import { once } from "@cocalc/util/async-utils"; +import { before, after, wait } from "@cocalc/backend/conat/test/setup"; + +beforeAll(before); + +describe("test headers with a dstream", () => { + let s; + const name = `${Math.random()}`; + it("creates a dstream and writes a value without a header", async () => { + s = await dstream({ name }); + expect(s.headers(s.length - 1)).toBe(undefined); + s.publish("x"); + await once(s, "change"); + const h = s.headers(s.length - 1); + for (const k in h ?? {}) { + if (!k.startsWith("CN-")) { + throw Error("system headers must start with CN-"); + } + } + }); + + it("writes a value with a header", async () => { + s.publish("y", { headers: { my: "header" } }); + // header isn't visible until ack'd by server + // NOTE: not optimal but this is what is implemented and documented! + expect(s.headers(s.length - 1)).toEqual(undefined); + await wait({ until: () => s.headers(s.length - 1) != null }); + expect(s.headers(s.length - 1)).toEqual( + expect.objectContaining({ my: "header" }), + ); + }); + + it("header still there", async () => { + await s.close(); + s = await dstream({ name }); + expect(s.headers(s.length - 1)).toEqual( + expect.objectContaining({ my: "header" }), + ); + }); + + it("clean up", async () => { + await s.delete({ all: true }); + }); +}); + +describe("test headers with a dkv", () => { + let s; + const name = `${Math.random()}`; + it("creates a dkv and writes a value without a header", async () => { + s = await dkv({ name }); + s.set("x", 10); + await once(s, "change"); + const h = s.headers("x"); + for (const k in h ?? {}) { + if (!k.startsWith("CN-")) { + throw Error("system headers must start with CN-"); + } + } + }); + + it("writes a value with a header - defined even before saving", async () => { + s.set("y", 20, { headers: { my: "header" } }); + expect(s.headers("y")).toEqual(expect.objectContaining({ my: "header" })); + }); + + it("clean up", async () => { + await s.clear(); + }); +}); + +afterAll(after); diff --git a/src/packages/backend/conat/test/sync/inventory.test.ts b/src/packages/backend/conat/test/sync/inventory.test.ts new file mode 100644 index 0000000000..b345f3aafc --- /dev/null +++ b/src/packages/backend/conat/test/sync/inventory.test.ts @@ -0,0 +1,120 @@ +/* +Testing basic ops with dkv + +DEVELOPMENT: + +pnpm test ./inventory.test.ts + +*/ + +import { before, after, client } from "@cocalc/backend/conat/test/setup"; + +beforeAll(before); + +describe("test the (partial) inventory method on dkv", () => { + let dkv; + const name = `inventory-dkv`; + + it("creates a kv and grabs the partial inventory", async () => { + dkv = await client.sync.dkv({ name }); + const i = await dkv.kv.inventory(); + expect(i).toEqual({ + bytes: 0, + count: 0, + limits: { + allow_msg_ttl: true, + }, + seq: 0, + }); + }); + + it("set an element and see that updated in the inventory data", async () => { + dkv.a = 5; + const i = await dkv.kv.inventory(); + expect(i).toEqual({ + bytes: 2, + count: 1, + limits: { + allow_msg_ttl: true, + }, + seq: 1, + }); + }); + + it("delete an element and see that count does NOT change, because of the tombstone; bytes are larger though since it has to contain the tombstone (in a header)", async () => { + delete dkv.a; + const { bytes, count, seq } = await dkv.kv.inventory(); + expect({ bytes, count, seq }).toEqual({ + bytes: 23, + count: 1, + seq: 2, + }); + }); + + it("change some limits", async () => { + await dkv.config({ max_age: 100000, max_bytes: 100, max_msg_size: 100 }); + const { limits } = await dkv.kv.inventory(); + expect(limits).toEqual({ + allow_msg_ttl: true, + max_age: 100000, + max_bytes: 100, + max_msg_size: 100, + }); + }); +}); + +describe("test the (partial) inventory method on a dstream", () => { + let dstream; + const name = `inventory-dstream`; + + it("creates a dstream and grabs the partial inventory", async () => { + dstream = await client.sync.dstream({ name }); + const i = await dstream.stream.inventory(); + expect(i).toEqual({ + bytes: 0, + count: 0, + limits: {}, + seq: 0, + }); + }); + + it("publish see that updated in the inventory data", async () => { + dstream.publish(5); + await dstream.save(); + const i = await dstream.stream.inventory(); + expect(i).toEqual({ + bytes: 1, + count: 1, + limits: {}, + seq: 1, + }); + }); + + it("publish some more", async () => { + dstream.push(1, 2, 3, 4); + await dstream.save(); + const i = await dstream.stream.inventory(); + expect(i).toEqual({ + bytes: 5, + count: 5, + limits: {}, + seq: 5, + }); + }); + + it("change some limits", async () => { + await dstream.config({ + max_age: 100000, + max_bytes: 100, + max_msg_size: 100, + }); + const { limits } = await dstream.stream.inventory(); + expect(limits).toEqual({ + max_age: 100000, + max_bytes: 100, + max_msg_size: 100, + }); + }); +}); + +afterAll(after); diff --git a/src/packages/backend/conat/test/sync/limits.test.ts b/src/packages/backend/conat/test/sync/limits.test.ts new file mode 100644 index 0000000000..7aa2df2820 --- /dev/null +++ b/src/packages/backend/conat/test/sync/limits.test.ts @@ -0,0 +1,470 @@ +/* +Testing the limits. + +DEVELOPMENT: + +pnpm test ./limits.test.ts + +*/ + +import { dkv as createDkv } from "@cocalc/backend/conat/sync"; +import { dstream as createDstream } from "@cocalc/backend/conat/sync"; +import { delay } from "awaiting"; +import { once } from "@cocalc/util/async-utils"; +import { + before, + after, + wait, + connect, + client, +} from "@cocalc/backend/conat/test/setup"; + +beforeAll(before); + +describe("create a dkv with limit on the total number of keys, and confirm auto-delete works", () => { + let kv; + const name = `test-${Math.random()}`; + + it("creates the dkv", async () => { + kv = await createDkv({ client, name, config: { max_msgs: 2 } }); + expect(kv.getAll()).toEqual({}); + }); + + it("adds 2 keys, then a third, and sees first is gone", async () => { + kv.a = 10; + kv.b = 20; + expect(kv.a).toEqual(10); + expect(kv.b).toEqual(20); + kv.c = 30; + expect(kv.c).toEqual(30); + // have to wait until it's all saved and acknowledged before enforcing limit + if (!kv.isStable()) { + await once(kv, "stable"); + } + // next change is the enforcement happening + if (kv.has("a")) { + await once(kv, "change", 500); + } + // and confirm it + expect(kv.a).toBe(undefined); + expect(kv.getAll()).toEqual({ b: 20, c: 30 }); + }); + + it("closes the kv", async () => { + await kv.clear(); + await kv.close(); + }); +}); + +describe("create a dkv with limit on age of keys, and confirm auto-delete works", () => { + let kv; + const name = `test-${Math.random()}`; + + it("creates the dkv", async () => { + kv = await createDkv({ client, name, config: { max_age: 50 } }); + expect(kv.getAll()).toEqual({}); + }); + + it("adds 2 keys, then a third, and sees first two are gone due to aging out", async () => { + kv.a = 10; + kv.b = 20; + expect(kv.a).toEqual(10); + expect(kv.b).toEqual(20); + await kv.save(); + await kv.config(); + await delay(50); + await kv.config(); + await delay(10); + expect(kv.has("a")).toBe(false); + expect(kv.has("b")).toBe(false); + }); + + it("closes the kv", async () => { + await kv.clear(); + await kv.close(); + }); +}); + +describe("create a dkv with limit on total bytes of keys, and confirm auto-delete works", () => { + let kv; + const name = `test-${Math.random()}`; + + it("creates the dkv", async () => { + kv = await createDkv({ client, name, config: { max_bytes: 100 } }); + expect(kv.getAll()).toEqual({}); + }); + + it("adds a key, then a second, and sees first one is gone due to bytes", async () => { + kv.a = "x".repeat(50); + kv.b = "x".repeat(55); + expect(kv.getAll()).toEqual({ a: "x".repeat(50), b: "x".repeat(55) }); + await kv.save(); + expect(kv.has("b")).toBe(true); + await wait({ + until: async () => { + await kv.config(); + return !kv.has("a"); + }, + }); + expect(kv.getAll()).toEqual({ b: "x".repeat(55) }); + }); + + it("closes the kv", async () => { + await kv.clear(); + await kv.close(); + }); +}); + +describe("create a dkv with limit on max_msg_size, and confirm writing small messages works but writing a big one result in a 'reject' event", () => { + let kv; + const name = `test-${Math.random()}`; + + it("creates the dkv", async () => { + kv = await createDkv({ client, name, config: { max_msg_size: 100 } }); + expect(kv.getAll()).toEqual({}); + }); + + it("adds a key, then a second big one results in a 'reject' event", async () => { + const rejects: { key: string; value: string }[] = []; + kv.once("reject", (x) => { + rejects.push(x); + }); + kv.a = "x".repeat(50); + await kv.save(); + kv.b = "x".repeat(150); + await kv.save(); + expect(rejects).toEqual([{ key: "b", value: "x".repeat(150) }]); + expect(kv.has("b")).toBe(false); + }); + + it("closes the kv", async () => { + await kv.clear(); + await kv.close(); + }); +}); + +describe("create a dstream with limit on the total number of messages, and confirm max_msgs, max_age works", () => { + let s, s2; + const name = `test-${Math.random()}`; + + it("creates the dstream and another with a different client", async () => { + s = await createDstream({ client, name, config: { max_msgs: 2 } }); + s2 = await createDstream({ + client: connect(), + name, + config: { max_msgs: 2 }, + noCache: true, + }); + expect(s.get()).toEqual([]); + expect((await s.config()).max_msgs).toBe(2); + expect((await s2.config()).max_msgs).toBe(2); + }); + + it("push 2 messages, then a third, and see first is gone and that this is reflected on both clients", async () => { + expect((await s.config()).max_msgs).toBe(2); + expect((await s2.config()).max_msgs).toBe(2); + s.push("a"); + s.push("b"); + await wait({ until: () => s.length == 2 && s2.length == 2 }); + expect(s2.get()).toEqual(["a", "b"]); + s.push("c"); + await wait({ + until: () => + s.get(0) != "a" && + s.get(1) == "c" && + s2.get(0) != "a" && + s2.get(1) == "c", + }); + expect(s.getAll()).toEqual(["b", "c"]); + expect(s2.getAll()).toEqual(["b", "c"]); + + // also check limits ar enforced if we close, then open new one: + await s.close(); + s = await createDstream({ client, name, config: { max_msgs: 2 } }); + expect(s.getAll()).toEqual(["b", "c"]); + + await s.config({ max_msgs: -1 }); + }); + + it("verifies that max_age works", async () => { + await s.save(); + expect(s.hasUnsavedChanges()).toBe(false); + await delay(300); + s.push("new"); + await s.config({ max_age: 20 }); // anything older than 20ms should be deleted + await wait({ until: () => s.length == 1 }); + expect(s.getAll()).toEqual(["new"]); + await s.config({ max_age: -1 }); + }); + + it("verifies that ttl works", async () => { + const conf = await s.config(); + expect(conf.allow_msg_ttl).toBe(false); + const conf2 = await s.config({ max_age: -1, allow_msg_ttl: true }); + expect(conf2.allow_msg_ttl).toBe(true); + + s.publish("ttl-message", { ttl: 50 }); + await s.save(); + await wait({ + until: async () => { + await s.config(); + return s.length == 1; + }, + }); + expect(s.get()).toEqual(["new"]); + }); + + it("verifies that max_bytes works -- publishing something too large causes everything to end up gone", async () => { + const conf = await s.config({ max_bytes: 100 }); + expect(conf.max_bytes).toBe(100); + s.publish("x".repeat(1000)); + await s.config(); + await wait({ until: () => s.length == 0 }); + expect(s.length).toBe(0); + }); + + it("max_bytes -- publish something then another thing that causes the first to get deleted", async () => { + s.publish("x".repeat(75)); + s.publish("y".repeat(90)); + await wait({ + until: async () => { + await s.config(); + return s.length == 1; + }, + }); + expect(s.get()).toEqual(["y".repeat(90)]); + await s.config({ max_bytes: -1 }); + }); + + it("verifies that max_msg_size rejects messages that are too big", async () => { + await s.config({ max_msg_size: 100 }); + expect((await s.config()).max_msg_size).toBe(100); + s.publish("x".repeat(70)); + await expect(async () => { + await s.stream.publish("x".repeat(150)); + }).rejects.toThrowError("max_msg_size"); + await s.config({ max_msg_size: 200 }); + s.publish("x".repeat(150)); + await s.config({ max_msg_size: -1 }); + expect((await s.config()).max_msg_size).toBe(-1); + }); + + it("closes the stream", async () => { + await s.close(); + await s2.close(); + }); +}); + +describe("create a dstream with limit on max_age, and confirm auto-delete works", () => { + let s; + const name = `test-${Math.random()}`; + + it("creates the dstream", async () => { + s = await createDstream({ client, name, config: { max_age: 50 } }); + }); + + it("push a message, then another and see first disappears", async () => { + s.push({ a: 10 }); + await delay(75); + s.push({ b: 20 }); + expect(s.get()).toEqual([{ a: 10 }, { b: 20 }]); + await wait({ + until: async () => { + await s.config(); + return s.length == 1; + }, + }); + expect(s.getAll()).toEqual([{ b: 20 }]); + }); + + it("closes the stream", async () => { + await s.delete({ all: true }); + await s.close(); + }); +}); + +describe("create a dstream with limit on max_bytes, and confirm auto-delete works", () => { + let s; + const name = `test-${Math.random()}`; + + it("creates the dstream", async () => { + // note: 60 and not 40 due to slack for headers + s = await createDstream({ client, name, config: { max_bytes: 60 } }); + }); + + it("push a message, then another and see first disappears", async () => { + s.push("x".repeat(40)); + s.push("x".repeat(45)); + s.push("x"); + if (!s.isStable()) { + await once(s, "stable"); + } + expect(s.getAll()).toEqual(["x".repeat(45), "x"]); + }); + + it("closes the stream", async () => { + await s.delete({ all: true }); + await s.close(); + }); +}); + +describe("create a dstream with limit on max_msg_size, and confirm auto-delete works", () => { + let s; + const name = `test-${Math.random()}`; + + it("creates the dstream", async () => { + s = await createDstream({ client, name, config: { max_msg_size: 50 } }); + }); + + it("push a message, then another and see first disappears", async () => { + const rejects: any[] = []; + s.on("reject", ({ mesg }) => { + rejects.push(mesg); + }); + s.push("x".repeat(40)); + s.push("y".repeat(60)); // silently vanishes (well a reject event is emitted) + s.push("x"); + await wait({ + until: async () => { + await s.config(); + return s.length == 2; + }, + }); + expect(s.getAll()).toEqual(["x".repeat(40), "x"]); + expect(rejects).toEqual(["y".repeat(60)]); + }); + + it("closes the stream", async () => { + await s.close(); + }); +}); + +describe("test discard_policy 'new' where writes are rejected rather than old data being deleted, for max_bytes and max_msgs", () => { + let s; + const name = `test-${Math.random()}`; + + it("creates the dstream", async () => { + s = await createDstream({ + client, + name, + // we can write at most 300 bytes and 3 messages. beyond that we + // get reject events. + config: { + max_bytes: 300, + max_msgs: 3, + discard_policy: "new", + desc: { example: "config" }, + }, + }); + const rejects: any[] = []; + s.on("reject", ({ mesg }) => { + rejects.push(mesg); + }); + s.publish("x"); + s.publish("y"); + s.publish("w"); + s.publish("foo"); + + await wait({ + until: async () => { + await s.config(); + return rejects.length == 1; + }, + }); + expect(s.getAll()).toEqual(["x", "y", "w"]); + expect(rejects).toEqual(["foo"]); + + s.publish("x".repeat(299)); + await wait({ + until: async () => { + await s.config(); + return rejects.length == 2; + }, + }); + expect(s.getAll()).toEqual(["x", "y", "w"]); + expect(rejects).toEqual(["foo", "x".repeat(299)]); + }); + + it("check the config is persisted", async () => { + const lastConfig = await s.config(); + s.close(); + s = await createDstream({ + client, + name, + noCache: true, + }); + const config = await s.config(); + expect(lastConfig).toEqual(config); + expect(lastConfig.desc).toEqual({ example: "config" }); + }); + + it("closes the stream", async () => { + s.close(); + }); +}); + +describe("test rate limiting", () => { + let s; + const name = `test-${Math.random()}`; + + it("creates the dstream", async () => { + s = await createDstream({ + client, + name, + // we can write at most 300 bytes and 3 messages. beyond that we + // get reject events. + config: { + max_bytes_per_second: 300, + max_msgs_per_second: 3, + discard_policy: "new", + }, + }); + const rejects: any[] = []; + s.on("reject", ({ mesg }) => { + rejects.push(mesg); + }); + }); + + it("closes the stream", async () => { + await s.close(); + }); +}); + +import { EPHEMERAL_MAX_BYTES } from "@cocalc/conat/persist/storage"; +describe(`ephemeral streams always have a hard cap of ${EPHEMERAL_MAX_BYTES} on max_bytes `, () => { + let s; + it("creates a non-ephemeral dstream and checks no automatic max_bytes set", async () => { + const s1 = await createDstream({ + client, + name: "test-NON-ephemeral", + ephemeral: false, + }); + expect((await s1.config()).max_bytes).toBe(-1); + s1.close(); + }); + + it("creates an ephemeral dstream and checks max bytes automatically set", async () => { + s = await createDstream({ + client, + name: "test-ephemeral", + ephemeral: true, + }); + expect((await s.config()).max_bytes).toBe(EPHEMERAL_MAX_BYTES); + }); + + it("trying to set larger doesn't work", async () => { + expect( + (await s.config({ max_bytes: 2 * EPHEMERAL_MAX_BYTES })).max_bytes, + ).toBe(EPHEMERAL_MAX_BYTES); + expect((await s.config()).max_bytes).toBe(EPHEMERAL_MAX_BYTES); + }); + + it("setting it smaller is allowed", async () => { + expect( + (await s.config({ max_bytes: EPHEMERAL_MAX_BYTES / 2 })).max_bytes, + ).toBe(EPHEMERAL_MAX_BYTES / 2); + expect((await s.config()).max_bytes).toBe(EPHEMERAL_MAX_BYTES / 2); + }); +}); + +afterAll(after); diff --git a/src/packages/backend/nats/test/sync/open-files.test.ts b/src/packages/backend/conat/test/sync/open-files.test.ts similarity index 78% rename from src/packages/backend/nats/test/sync/open-files.test.ts rename to src/packages/backend/conat/test/sync/open-files.test.ts index 47e05c6171..371ba6a476 100644 --- a/src/packages/backend/nats/test/sync/open-files.test.ts +++ b/src/packages/backend/conat/test/sync/open-files.test.ts @@ -8,13 +8,16 @@ to open so they can fulfill their backend responsibilities: DEVELOPMENT: -pnpm exec jest --forceExit "open-files.test.ts" +pnpm test ./open-files.test.ts */ -import { openFiles as createOpenFiles } from "@cocalc/backend/nats/sync"; +import { openFiles as createOpenFiles } from "@cocalc/backend/conat/sync"; import { once } from "@cocalc/util/async-utils"; import { delay } from "awaiting"; +import { before, after, wait } from "@cocalc/backend/conat/test/setup"; + +beforeAll(before); const project_id = "00000000-0000-4000-8000-000000000000"; async function create() { @@ -30,7 +33,7 @@ describe("create open file tracker and do some basic operations", () => { o1 = await create(); o2 = await create(); // ensure caching disabled so our sync tests are real - expect(o1.getDkv() === o2.getDkv()).toBe(false); + expect(o1.getKv() === o2.getKv()).toBe(false); o1.clear(); await o1.save(); expect(o1.hasUnsavedChanges()).toBe(false); @@ -74,33 +77,28 @@ describe("create open file tracker and do some basic operations", () => { expect(v.length).toBe(2); }); - it("delete file1 and verify that it is deleted is sync'd", async () => { + it("delete file1 and verify fact that it is deleted is sync'd", async () => { o1.delete(file1); expect(o1.get(file1)).toBe(undefined); expect(o1.getAll().length).toBe(1); await o1.save(); - // TODO/warning/weird: If I comment out this use of o3, - // then the test involving o2 below doesn't work sometimes, i.e., for some reason - // sync isn't working with the delete for two dkv's in the same process at once. - // (In practice our sync is between completely different clients, so I'm not - // super worried about this.) - // in a newly opened tracker, the file is clearly gone: - const o3 = await create(); - expect(o3.get(file1)).toBe(undefined); - // should be 1 due to file2 still being there: - expect(o3.getAll().length).toBe(1); - o3.close(); - - // verify file is gone in o2, at least after change event (if necessary) - if (o2.get(file1) !== undefined) { - console.log("wait for o2 change"); - await once(o2, "change", 250); - } + // verify file is gone in o2, at least after waiting (if necessary) + await wait({ + until: () => { + return o2.getAll().length == 1; + }, + }); expect(o2.get(file1)).toBe(undefined); // should be 1 due to file2 still being there: expect(o2.getAll().length).toBe(1); + // Also confirm file1 is gone in a newly opened one: + const o3 = await create(); + expect(o3.get(file1)).toBe(undefined); + // should be 1 due to file2 still being there, but not file1. + expect(o3.getAll().length).toBe(1); + o3.close(); }); it("sets an error", async () => { @@ -130,3 +128,5 @@ describe("create open file tracker and do some basic operations", () => { expect(o2.get(file2).error).toBe(undefined); }); }); + +afterAll(after); diff --git a/src/packages/backend/nats/test/sync/util.ts b/src/packages/backend/conat/test/sync/util.ts similarity index 81% rename from src/packages/backend/nats/test/sync/util.ts rename to src/packages/backend/conat/test/sync/util.ts index 9eed813594..e52641928b 100644 --- a/src/packages/backend/nats/test/sync/util.ts +++ b/src/packages/backend/conat/test/sync/util.ts @@ -1,4 +1,4 @@ -import { dstream } from "@cocalc/backend/nats/sync"; +import { dstream } from "@cocalc/backend/conat/sync"; export async function createDstream() { const name = `test-${Math.random()}`; @@ -11,6 +11,5 @@ export async function createDstreamEphemeral() { name, noAutosave: true, ephemeral: true, - leader: true, }); } diff --git a/src/packages/backend/nats/test/time.test.ts b/src/packages/backend/conat/test/time.test.ts similarity index 77% rename from src/packages/backend/nats/test/time.test.ts rename to src/packages/backend/conat/test/time.test.ts index 3386e79a76..4d2f7d3102 100644 --- a/src/packages/backend/nats/test/time.test.ts +++ b/src/packages/backend/conat/test/time.test.ts @@ -4,12 +4,13 @@ DEVELOPMENT: pnpm test ./time.test.ts */ -// this sets client -import "@cocalc/backend/nats"; +import { timeClient, createTimeService } from "@cocalc/conat/service/time"; +import time, { getSkew } from "@cocalc/conat/time"; +import { before, after } from "@cocalc/backend/conat/test/setup"; -import time, { getSkew } from "@cocalc/nats/time"; +beforeAll(before); -describe("get time from nats", () => { +describe("get time from conat", () => { it("tries to get the time before the skew, so it is not initialized yet", () => { expect(time).toThrow("clock skew not known"); }); @@ -29,14 +30,13 @@ describe("get time from nats", () => { }); }); -import { timeClient, createTimeService } from "@cocalc/nats/service/time"; - describe("start the time server and client and test that it works", () => { it("starts the time server and queries it", async () => { - await import("@cocalc/backend/nats"); createTimeService(); const client = timeClient(); const t = await client.time(); expect(Math.abs(Date.now() - t)).toBeLessThan(200); }); }); + +afterAll(after); diff --git a/src/packages/backend/conat/test/util.ts b/src/packages/backend/conat/test/util.ts new file mode 100644 index 0000000000..55f9be131d --- /dev/null +++ b/src/packages/backend/conat/test/util.ts @@ -0,0 +1,30 @@ +import { until } from "@cocalc/util/async-utils"; + +export async function wait({ + until: f, + start = 5, + decay = 1.2, + max = 300, +}: { + until: Function; + start?: number; + decay?: number; + max?: number; +}) { + await until( + async () => { + try { + return !!(await f()); + } catch { + return false; + } + }, + { + start, + decay, + max, + min: 5, + timeout: 10000, + }, + ); +} diff --git a/src/packages/backend/data.ts b/src/packages/backend/data.ts index 5151004969..fbde609b38 100644 --- a/src/packages/backend/data.ts +++ b/src/packages/backend/data.ts @@ -28,7 +28,8 @@ import { join, resolve } from "path"; import { ConnectionOptions } from "node:tls"; import { existsSync, mkdirSync, readFileSync } from "fs"; import { isEmpty } from "lodash"; -import { hostname } from "os"; +import basePath from "@cocalc/backend/base-path"; +import port from "@cocalc/backend/port"; function determineRootFromPath(): string { const cur = __dirname; @@ -179,6 +180,15 @@ export const pgdatabase: string = export const projects: string = process.env.PROJECTS ?? join(data, "projects", "[project_id]"); export const secrets: string = process.env.SECRETS ?? join(data, "secrets"); + +export const syncFiles = { + // Persistent local storage of streams and kv's as sqlite3 files + local: process.env.COCALC_SYNC ?? join(data, "sync"), + // Archived storage of streams and kv's as sqlite3 files, if set. + // This could be a gcsfuse mountpoint. + archive: process.env.COCALC_SYNC_ARCHIVE ?? "", +}; + // if the directory secrets doesn't exist, create it (sync, during this load): if (!existsSync(secrets)) { try { @@ -191,110 +201,67 @@ if (!existsSync(secrets)) { export const logs: string = process.env.LOGS ?? join(data, "logs"); -// NATS -export const nats: string = process.env.COCALC_NATS ?? join(data, "nats"); -if (!existsSync(nats)) { - try { - mkdirSync(nats, { recursive: true, mode: 0o700 }); - } catch { - // nonfatal -- there are other ways to auth to nats - } +// CONAT server and password +export let conatServer = + process.env.CONAT_SERVER ?? + `http://localhost:${port}${basePath.length > 1 ? basePath : ""}`; +if (conatServer.split("//").length > 2) { + // i make this mistake too much + throw Error( + `env variable CONAT_SERVER invalid -- too many /s' --'${process.env.CONAT_SERVER}'`, + ); } -export const natsPorts = { - server: parseInt(process.env.COCALC_NATS_PORT ?? "4222"), - ws: parseInt(process.env.COCALC_NATS_WS_PORT ?? "8443"), - cluster: parseInt(process.env.COCALC_NATS_CLUSTER_PORT ?? "4248"), -}; - -export const natsServerName = process.env.COCALC_NATS_SERVER_NAME ?? hostname(); -export const natsClusterName = - process.env.COCALC_NATS_CLUSTER_NAME ?? "default"; -export let natsServer = process.env.COCALC_NATS_SERVER ?? "localhost"; -// note: natsWebsocketServer will be changed below if API_KEY and API_SERVER -// are set, but COCALC_NATS_SERVER is not set. -export let natsWebsocketServer = `ws://${natsServer}:${natsPorts.ws}`; - -export function setNatsPort(port) { - natsPorts.server = parseInt(port); -} -export function setNatsWebsocketPort(port) { - natsPorts.ws = parseInt(port); - natsWebsocketServer = `ws://${natsServer}:${natsPorts.ws}`; -} -export function setNatsServer(server) { - natsServer = server; - natsWebsocketServer = `ws://${natsServer}:${natsPorts.ws}`; +export function setConatServer(server: string) { + conatServer = server; } -// Password used to connect to the nats server -export let natsPassword = ""; -export const natsPasswordPath = join(secrets, "nats_password"); +// Password used by hub (not users or projects) to connect to a Conat server: +export let conatPassword = ""; +export const conatPasswordPath = join(secrets, "conat-password"); try { - natsPassword = readFileSync(natsPasswordPath).toString().trim(); + conatPassword = readFileSync(conatPasswordPath).toString().trim(); } catch {} -export function setNatsPassword(password: string) { - natsPassword = password; +export function setConatPassword(password: string) { + conatPassword = password; } -export const natsBackup = - process.env.COCALC_NATS_BACKUP ?? join(nats, "backup"); - -export const natsUser = "cocalc"; - -// Secrets used for cryptography between the auth callout service and -// and the nats server. The *secret keys* are only needed by -// the auth callout service, and the corresponding public keys are -// only needed by the nats server, but right now (and since password is already -// known to both), we are just making the private keys available to both. -// These keys make it so if somebody tries to listen in on nats traffic -// between the server and auth callout, they can't impersonate users, etc. -// In particular: -// - nseed = account key - used by server to sign message to the auth callout -// - xseed = curve key - used by auth callout to encrypt response -// These are both arbitrary elliptic curve ed25519 secrets (nkeys), -// which are the "seed" generated using https://www.npmjs.com/package/@nats-io/nkeys -// or https://github.com/nats-io/nkeys?tab=readme-ov-file#installation -// E.g., -// ~/cocalc/src/data/secrets$ go get github.com/nats-io/nkeys -// ~/cocalc/src/data/secrets$ nk -gen account > nats_auth_nseed -// ~/cocalc/src/data/secrets$ nk -gen curve > nats_auth_xseed - -export let natsAuthCalloutNSeed = ""; -export const natsAuthCalloutNSeedPath = join(secrets, "nats_auth_nseed"); -try { - natsAuthCalloutNSeed = readFileSync(natsAuthCalloutNSeedPath) - .toString() - .trim(); -} catch {} -export function setNatsAuthCalloutNSeed(auth_callout: string) { - natsAuthCalloutNSeed = auth_callout; +export let conatValkey: string = process.env.CONAT_VALKEY ?? ""; +export function setConatValkey(valkey: string) { + conatValkey = valkey; } -export let natsAuthCalloutXSeed = ""; -export const natsAuthCalloutXSeedPath = join(secrets, "nats_auth_xseed"); + +export let valkeyPassword = ""; +const valkeyPasswordPath = join(secrets, "valkey-password"); try { - natsAuthCalloutXSeed = readFileSync(natsAuthCalloutXSeedPath) - .toString() - .trim(); + valkeyPassword = readFileSync(valkeyPasswordPath).toString().trim(); } catch {} -export function setNatsAuthCalloutXSeed(auth_callout: string) { - natsAuthCalloutXSeed = auth_callout; -} + +export let conatSocketioCount = parseInt( + process.env.CONAT_SOCKETIO_COUNT ?? "1", +); + +// number of persist servers (if configured to run) +export let conatPersistCount = parseInt(process.env.CONAT_PERSIST_COUNT ?? "1"); + +// number of api servers (if configured to run) +export let conatApiCount = parseInt(process.env.CONAT_API_COUNT ?? "1"); + +// if configured, will create a socketio cluster using +// the cluster adapter, listening on the given port. +// It makes no sense to use both this *and* valkey. It's +// one or the other. +export let conatClusterPort = parseInt(process.env.CONAT_CLUSTER_PORT ?? "0"); +// if set, a simple http server will be started listening on conatClusterHealthPort +// which returns an error only if the socketio server is not "healthy". +export let conatClusterHealthPort = parseInt( + process.env.CONAT_CLUSTER_HEALTH_PORT ?? "0", +); // API keys export let apiKey: string = process.env.API_KEY ?? ""; export let apiServer: string = process.env.API_SERVER ?? ""; -if ( - process.env.API_KEY && - process.env.API_SERVER && - !process.env.COCALC_NATS_SERVER -) { - // the nats server was not set via env variables, but the api server is set, - // along with the api key. This happens for compute servers, and in this case - // we also set the nats server by default to the same as the api server. - natsWebsocketServer = "ws" + apiServer.slice(4) + "/nats"; -} // Delete API_KEY from environment to reduce chances of it leaking, e.g., to // spawned terminal subprocess. diff --git a/src/packages/backend/execute-code.test.ts b/src/packages/backend/execute-code.test.ts index d8a3b1db4f..8cca0e920d 100644 --- a/src/packages/backend/execute-code.test.ts +++ b/src/packages/backend/execute-code.test.ts @@ -7,15 +7,17 @@ DEVELOPMENT: -pnpm exec jest --watch --forceExit --detectOpenHandles "execute-code.test.ts" +pnpm test ./execute-code.test.ts */ +import { delay } from "awaiting"; + process.env.COCALC_PROJECT_MONITOR_INTERVAL_S = "1"; // default is much lower, might fail if you have more procs than the default process.env.COCALC_PROJECT_INFO_PROC_LIMIT = "10000"; -import { executeCode } from "./execute-code"; +import { executeCode, setMonitorIntervalSeconds } from "./execute-code"; describe("hello world", () => { it("runs hello world", async () => { @@ -159,7 +161,7 @@ describe("async", () => { expect(start).toBeGreaterThan(1); expect(typeof job_id).toEqual("string"); if (typeof job_id !== "string") return; - await new Promise((done) => setTimeout(done, 250)); + await delay(250); { const s = await executeCode({ async_get: job_id }); expect(s.type).toEqual("async"); @@ -172,7 +174,7 @@ describe("async", () => { expect(s.exit_code).toEqual(0); } - await new Promise((done) => setTimeout(done, 900)); + await delay(900); { const s = await executeCode({ async_get: job_id }); expect(s.type).toEqual("async"); @@ -199,7 +201,7 @@ describe("async", () => { const { job_id } = c; expect(typeof job_id).toEqual("string"); if (typeof job_id !== "string") return; - await new Promise((done) => setTimeout(done, 250)); + await delay(250); const s = await executeCode({ async_get: job_id }); expect(s.type).toEqual("async"); if (s.type !== "async") return; @@ -223,7 +225,7 @@ describe("async", () => { const { job_id } = c; expect(typeof job_id).toEqual("string"); if (typeof job_id !== "string") return; - await new Promise((done) => setTimeout(done, 250)); + await delay(250); const s = await executeCode({ async_get: job_id }); expect(s.type).toEqual("async"); if (s.type !== "async") return; @@ -248,7 +250,7 @@ describe("async", () => { expect(start).toBeGreaterThan(1); expect(typeof job_id).toEqual("string"); if (typeof job_id !== "string") return; - await new Promise((done) => setTimeout(done, 250)); + await delay(250); // now we check up on the job const s = await executeCode({ async_get: job_id }); expect(s.type).toEqual("async"); @@ -264,54 +266,49 @@ describe("async", () => { expect(s.exit_code).toEqual(1); }); - // TODO: I really don't like these tests, which waste a lot of my time. - // Instead of taking 5+ seconds to test some polling implementation, - // they should have a parameter to change the polling interval, so the - // test can be much quicker. -- WS - it( - "long running async job", - async () => { - const c = await executeCode({ - command: "sh", - args: ["-c", `echo foo; python3 -c '${CPU_PY}'; echo bar;`], - bash: false, - err_on_exit: false, - async_call: true, - }); - expect(c.type).toEqual("async"); - if (c.type !== "async") return; - const { status, job_id } = c; - expect(status).toEqual("running"); - expect(typeof job_id).toEqual("string"); - if (typeof job_id !== "string") return; - await new Promise((done) => setTimeout(done, 5500)); - // now we check up on the job - const s = await executeCode({ async_get: job_id, async_stats: true }); - expect(s.type).toEqual("async"); - if (s.type !== "async") return; - expect(s.elapsed_s).toBeGreaterThan(5); - expect(s.exit_code).toBe(0); - expect(s.pid).toBeGreaterThan(1); - expect(s.stats).toBeDefined(); - if (!Array.isArray(s.stats)) return; - const pcts = Math.max(...s.stats.map((s) => s.cpu_pct)); - const secs = Math.max(...s.stats.map((s) => s.cpu_secs)); - const mems = Math.max(...s.stats.map((s) => s.mem_rss)); - expect(pcts).toBeGreaterThan(10); - expect(secs).toBeGreaterThan(1); - expect(mems).toBeGreaterThan(1); - expect(s.stdout).toEqual("foo\nbar\n"); - // now without stats, after retrieving it - const s2 = await executeCode({ async_get: job_id }); - if (s2.type !== "async") return; - expect(s2.stats).toBeUndefined(); - // and check, that this is not removing stats entirely - const s3 = await executeCode({ async_get: job_id, async_stats: true }); - if (s3.type !== "async") return; - expect(Array.isArray(s3.stats)).toBeTruthy(); - }, - 10 * 1000, - ); + // This test screws up running multiple tests in parallel. + // ** HENCE SKIPPING THIS - enable it if you edit the executeCode code...** + it.skip("longer running async job", async () => { + setMonitorIntervalSeconds(1); + const c = await executeCode({ + command: "sh", + args: ["-c", `echo foo; python3 -c '${CPU_PY}'; echo bar;`], + bash: false, + err_on_exit: false, + async_call: true, + }); + expect(c.type).toEqual("async"); + if (c.type !== "async") return; + const { status, job_id } = c; + expect(status).toEqual("running"); + expect(typeof job_id).toEqual("string"); + if (typeof job_id !== "string") return; + await delay(3000); + // now we check up on the job + const s = await executeCode({ async_get: job_id, async_stats: true }); + expect(s.type).toEqual("async"); + if (s.type !== "async") return; + expect(s.elapsed_s).toBeGreaterThan(1); + expect(s.exit_code).toBe(0); + expect(s.pid).toBeGreaterThan(1); + expect(s.stats).toBeDefined(); + if (!Array.isArray(s.stats)) return; + const pcts = Math.max(...s.stats.map((s) => s.cpu_pct)); + const secs = Math.max(...s.stats.map((s) => s.cpu_secs)); + const mems = Math.max(...s.stats.map((s) => s.mem_rss)); + expect(pcts).toBeGreaterThan(10); + expect(secs).toBeGreaterThan(1); + expect(mems).toBeGreaterThan(1); + expect(s.stdout).toEqual("foo\nbar\n"); + // now without stats, after retrieving it + const s2 = await executeCode({ async_get: job_id }); + if (s2.type !== "async") return; + expect(s2.stats).toBeUndefined(); + // and check, that this is not removing stats entirely + const s3 = await executeCode({ async_get: job_id, async_stats: true }); + if (s3.type !== "async") return; + expect(Array.isArray(s3.stats)).toBeTruthy(); + }); }); // the await case is essentially like the async case above, but it will block for a bit @@ -366,7 +363,7 @@ describe("await", () => { const { status, job_id, pid } = c; expect(status).toEqual("running"); expect(pid).toBeGreaterThan(1); - await new Promise((done) => setTimeout(done, 2000)); + await delay(2000); const s = await executeCode({ async_await: true, async_get: job_id, @@ -438,7 +435,7 @@ describe("await", () => { expect(c.type).toEqual("async"); if (c.type !== "async") return; const { job_id, pid } = c; - await new Promise((done) => setTimeout(done, 100)); + await delay(100); await executeCode({ command: `kill -9 -${pid}`, bash: true, @@ -461,6 +458,6 @@ describe("await", () => { const CPU_PY = ` from time import time t0=time() -while t0+5>time(): +while t0+2.5>time(): sum([_ for _ in range(10**6)]) `; diff --git a/src/packages/backend/execute-code.ts b/src/packages/backend/execute-code.ts index bf7acc5cd9..31b9904324 100644 --- a/src/packages/backend/execute-code.ts +++ b/src/packages/backend/execute-code.ts @@ -5,7 +5,7 @@ // Execute code in a subprocess. -import { callback } from "awaiting"; +import { callback, delay } from "awaiting"; import LRU from "lru-cache"; import { ChildProcessWithoutNullStreams, @@ -43,7 +43,12 @@ const PREFIX = "COCALC_PROJECT_ASYNC_EXEC"; const ASYNC_CACHE_MAX = envToInt(`${PREFIX}_CACHE_MAX`, 100); const ASYNC_CACHE_TTL_S = envToInt(`${PREFIX}_TTL_S`, 60 * 60); // for async execution, every that many secs check up on the child-tree -const MONITOR_INTERVAL_S = envToInt(`${PREFIX}_MONITOR_INTERVAL_S`, 60); +let MONITOR_INTERVAL_S = envToInt(`${PREFIX}_MONITOR_INTERVAL_S`, 60); + +export function setMonitorIntervalSeconds(n) { + MONITOR_INTERVAL_S = n; +} + const MONITOR_STATS_LENGTH_MAX = envToInt( `${PREFIX}_MONITOR_STATS_LENGTH_MAX`, 100, @@ -117,9 +122,13 @@ export const execute_code: ExecuteCodeFunctionWithCallback = aggregate( }, ); -async function clean_up_tmp(tempDir: string | undefined) { +export async function cleanUpTempDir(tempDir: string | undefined) { if (tempDir) { - await rm(tempDir, { force: true, recursive: true }); + try { + await rm(tempDir, { force: true, recursive: true }); + } catch (err) { + console.log("WARNING: issue cleaning up tempDir", err); + } } } @@ -171,6 +180,9 @@ async function executeCodeNoAggregate( } else if (opts.path[0] !== "/") { opts.path = opts.home + "/" + opts.path; } + if (opts.cwd) { + opts.path = opts.cwd; + } let tempDir: string | undefined = undefined; @@ -224,7 +236,7 @@ async function executeCodeNoAggregate( }; asyncCache.set(job_id, job_config); - const pid: number | undefined = doSpawn( + const child = doSpawn( { ...opts, origCommand, job_id, job_config }, async (err, result) => { log.debug("async/doSpawn returned", { err, result }); @@ -261,10 +273,11 @@ async function executeCodeNoAggregate( }); } } finally { - await clean_up_tmp(tempDir); + await cleanUpTempDir(tempDir); } }, ); + const pid = child?.pid; // pid could be undefined, this means it wasn't possible to spawn a child return { ...job_config, pid }; @@ -274,7 +287,9 @@ async function executeCodeNoAggregate( } } finally { // do not delete the tempDir in async mode! - if (!opts.async_call) await clean_up_tmp(tempDir); + if (!opts.async_call) { + await cleanUpTempDir(tempDir); + } } } @@ -307,8 +322,8 @@ function doSpawn( job_id?: string; job_config?: ExecuteCodeOutputAsync; }, - cb: (err: string | undefined, result?: ExecuteCodeOutputBlocking) => void, -): number | undefined { + cb?: (err: string | undefined, result?: ExecuteCodeOutputBlocking) => void, +) { const start_time = walltime(); if (opts.verbose) { @@ -355,7 +370,7 @@ function doSpawn( if (job_id == null || pid == null || job_config == null) return; const monitor = new ProcessStats(); await monitor.init(); - await new Promise((done) => setTimeout(done, 1000)); + await delay(1000); if (callback_done) return; while (true) { @@ -394,7 +409,7 @@ function doSpawn( // i.e. after 6 minutes, we check every minute const next_s = Math.max(1, Math.floor(elapsed_s / 6)); const wait_s = Math.min(next_s, MONITOR_INTERVAL_S); - await new Promise((done) => setTimeout(done, wait_s * 1000)); + await delay(wait_s * 1000); } } @@ -404,14 +419,14 @@ function doSpawn( // The docs/examples at https://nodejs.org/api/child_process.html#child_process_child_process_spawn_command_args_options // suggest that r.stdout and r.stderr are always defined. However, this is // definitely NOT the case in edge cases, as we have observed. - cb("error creating child process -- couldn't spawn child process"); + cb?.("error creating child process -- couldn't spawn child process"); return; } } catch (error) { // Yes, spawn can cause this error if there is no memory, and there's no // event! -- Error: spawn ENOMEM ran_code = false; - cb(`error ${error}`); + cb?.(`error ${error}`); return; } @@ -540,16 +555,16 @@ function doSpawn( } if (err) { - cb(err); + cb?.(err); } else if (opts.err_on_exit && exit_code != 0) { const x = opts.origCommand ? opts.origCommand : `'${opts.command}' (args=${opts.args?.join(" ")})`; if (opts.job_id) { - cb(stderr); + cb?.(stderr); } else { // sync behavor, like it was before - cb( + cb?.( `command '${x}' exited with nonzero code ${exit_code} -- stderr='${trunc( stderr, 1024, @@ -561,7 +576,7 @@ function doSpawn( const x = opts.origCommand ? opts.origCommand : `'${opts.command}' (args=${opts.args?.join(" ")})`; - cb( + cb?.( `command '${x}' was not able to run -- stderr='${trunc(stderr, 1024)}'`, ); } else { @@ -577,7 +592,7 @@ function doSpawn( // if exit-code not set, may have been SIGKILL so we set it to 1 exit_code = 1; } - cb(undefined, { type: "blocking", stdout, stderr, exit_code }); + cb?.(undefined, { type: "blocking", stdout, stderr, exit_code }); } }; @@ -611,5 +626,5 @@ function doSpawn( timer = setTimeout(f, opts.timeout * 1000); } - return child.pid; + return child; } diff --git a/src/packages/backend/get-port.test.ts b/src/packages/backend/get-port.test.ts new file mode 100644 index 0000000000..95e833bf1e --- /dev/null +++ b/src/packages/backend/get-port.test.ts @@ -0,0 +1,20 @@ +import getPort, { getPorts } from "./get-port"; + +describe("test getting a random available port", () => { + it("tests it", async () => { + const port = await getPort(); + expect(port).toBeGreaterThan(1024); + }); +}); + +describe("test getPorts -- getting many ports at once in parallel", () => { + const count = 1000; + it(`get ${count} ports at once, thus testing it isn't too slow and also nice to see no conflicts`, async () => { + const start = Date.now(); + const w = await getPorts(count); + expect(new Set(w).size).toBe(count); + // takes ~200ms to get 5000 of them on my laptop, but with heavy load + // it can be much slower. + expect(Date.now() - start).toBeLessThan(4000); + }); +}); diff --git a/src/packages/backend/get-port.ts b/src/packages/backend/get-port.ts new file mode 100644 index 0000000000..baa0db5ffc --- /dev/null +++ b/src/packages/backend/get-port.ts @@ -0,0 +1,25 @@ +import { createServer } from "http"; + +export default async function getPort(): Promise { + return new Promise((resolve, reject) => { + const server = createServer(); + server.listen(0, () => { + const address = server.address(); + if (typeof address === "object" && address !== null) { + const port = address.port; + server.close(() => resolve(port)); + } else { + reject(new Error("Failed to get port")); + } + }); + server.on("error", reject); + }); +} + +export async function getPorts(n: number): Promise { + const v: any[] = []; + for (let i = 0; i < n; i++) { + v.push(getPort()); + } + return Promise.all(v); +} diff --git a/src/packages/backend/logger.ts b/src/packages/backend/logger.ts index bc4a9ded88..6143471fcb 100644 --- a/src/packages/backend/logger.ts +++ b/src/packages/backend/logger.ts @@ -115,7 +115,7 @@ function initTransports() { transports.console ? " and console.log" : "" } via the debug module\nwith DEBUG='${ process.env.DEBUG - }'.\nUse DEBUG_FILE='path' and DEBUG_CONSOLE=[yes|no] to override.\nUsing DEBUG='cocalc:*,-cocalc:silly:*' to control log levels.\n\n***`; + }'.\nUse DEBUG_FILE='path' and DEBUG_CONSOLE=[yes|no] to override.\nUsing e.g., something like DEBUG='cocalc:*,-cocalc:silly:*' to control log levels.\n\n***`; console.log(announce); if (transports.file) { // the file transport diff --git a/src/packages/backend/misc.ts b/src/packages/backend/misc.ts index 22f9c00f34..c52b14a34f 100644 --- a/src/packages/backend/misc.ts +++ b/src/packages/backend/misc.ts @@ -69,3 +69,14 @@ export function envForSpawn() { } return env; } + +import { callback } from "awaiting"; +import { randomBytes } from "crypto"; + +export async function secureRandomString(length: number): Promise { + return (await callback(randomBytes, length)).toString("base64"); +} + +export function secureRandomStringSync(length: number): string { + return randomBytes(length).toString("base64"); +} diff --git a/src/packages/backend/misc/ensure-containing-directory-exists.ts b/src/packages/backend/misc/ensure-containing-directory-exists.ts index 322dbd67a9..75033abbc7 100644 --- a/src/packages/backend/misc/ensure-containing-directory-exists.ts +++ b/src/packages/backend/misc/ensure-containing-directory-exists.ts @@ -7,20 +7,23 @@ import abspath from "./abspath"; // Make sure that that the directory containing the file indicated by // the path exists and has restrictive permissions. export default async function ensureContainingDirectoryExists( - path: string + path: string, ): Promise { path = abspath(path); const containingDirectory = path_split(path).head; // containing path if (!containingDirectory) return; + await ensureDirectoryExists(containingDirectory); +} +export async function ensureDirectoryExists(path: string): Promise { try { - await access(containingDirectory, fsc.R_OK | fsc.W_OK); + await access(path, fsc.R_OK | fsc.W_OK); // it exists, yeah! return; } catch (err) { // Doesn't exist, so create, via recursion: try { - await mkdir(containingDirectory, { mode: 0o700, recursive: true }); + await mkdir(path, { mode: 0o700, recursive: true }); } catch (err) { if (err?.code === "EEXIST") { // no problem -- it exists. diff --git a/src/packages/backend/misc/new-file.ts b/src/packages/backend/misc/new-file.ts index 9954d8bd6c..5fd9b69f2d 100644 --- a/src/packages/backend/misc/new-file.ts +++ b/src/packages/backend/misc/new-file.ts @@ -10,6 +10,9 @@ export async function newFile(path: string) { if (!process.env.HOME) { throw Error("HOME must be set"); } + if (!path) { + return; + } path = path.startsWith("/") ? path : join(process.env.HOME, path); if (await exists(path)) { diff --git a/src/packages/backend/nats/cli.ts b/src/packages/backend/nats/cli.ts deleted file mode 100644 index 154eb5dc2e..0000000000 --- a/src/packages/backend/nats/cli.ts +++ /dev/null @@ -1,55 +0,0 @@ -/* -Run an interactive bash terminal, but with the nats and nsc command -available and configured to work with full permissions. This is -useful for interactively using those command to inspect the state -of the system, learning how to do something, etc. -*/ - -import { data, natsPassword, natsUser } from "@cocalc/backend/data"; -import { join } from "path"; -import { spawnSync } from "node:child_process"; -import { natsServerUrl } from "./conf"; - -const natsBin = join(data, "nats", "bin"); - -export function natsCoCalcUserEnv({ user = natsUser }: { user?: string } = {}) { - return { - NATS_URL: natsServerUrl, - NATS_PASSWORD: natsPassword, - NATS_USER: user ?? natsUser, - PATH: `${natsBin}:${process.env.PATH}`, - }; -} - -function params({ user }) { - return { - command: "bash", - args: ["--norc", "--noprofile"], - env: { - ...natsCoCalcUserEnv({ user }), - HOME: process.env.HOME, - TERM: process.env.TERM, - PS1: `\\w [nats-${user}]$ `, - }, - }; -} - -// echo; echo '# Use CoCalc config of NATS (nats and nsc) via this subshell:'; echo; NATS_URL=nats://${COCALC_NATS_SERVER:=localhost}:${COCALC_NATS_PORT:=4222} XDG_DATA_HOME=${COCALC_ROOT:=$INIT_CWD}/data XDG_CONFIG_HOME=${COCALC_ROOT:=$INIT_CWD}/data PATH=${COCALC_ROOT:=$INIT_CWD}/data/nats/bin:$PATH bash - -// the supported users here are natsUser and 'sys'. - -export function main({ user = natsUser }: { user?: string } = {}) { - let { command, args, env } = params({ user }); - console.log("# Use CoCalc config of NATS (nats and nsc) via this subshell:"); - console.log( - JSON.stringify( - { ...env, NATS_PASSWORD: "xxx", PATH: natsBin + ":..." }, - undefined, - 2, - ), - ); - spawnSync(command, args, { - env: { ...env, PATH: `${natsBin}:${process.env.PATH}` }, - stdio: "inherit", - }); -} diff --git a/src/packages/backend/nats/conf.ts b/src/packages/backend/nats/conf.ts deleted file mode 100644 index 3cd8e07d62..0000000000 --- a/src/packages/backend/nats/conf.ts +++ /dev/null @@ -1,181 +0,0 @@ -/* -Configure nats-server, i.e., generate configuration files. - -node -e "require('@cocalc/backend/nats/conf').main()" - - - -NOTES: - -- I tried very hard to use NKEYS and/or JWT, but it's -just not compatible with auth callout, and auth callout -is required for scalability, given my use case. That's -why there is an explicit password. -*/ - -import { pathExists } from "fs-extra"; -import { - nats, - natsPorts, - natsServer, - natsPassword, - natsPasswordPath, - setNatsPassword, - natsUser, - natsAuthCalloutNSeed, - setNatsAuthCalloutNSeed, - natsAuthCalloutNSeedPath, - natsAuthCalloutXSeed, - setNatsAuthCalloutXSeed, - natsAuthCalloutXSeedPath, - natsClusterName, - natsServerName, -} from "@cocalc/backend/data"; -import { join } from "path"; -import getLogger from "@cocalc/backend/logger"; -import { writeFile } from "fs/promises"; -import { REMEMBER_ME_COOKIE_NAME } from "@cocalc/backend/auth/cookie-names"; -import { executeCode } from "@cocalc/backend/execute-code"; -import { createPrivateKey, publicKey } from "./nkeys"; - -const logger = getLogger("backend:nats:install"); - -// this is assumed in cocalc/src/package.json: -const confPath = join(nats, "server.conf"); - -// for now for local dev: -export const natsServerUrl = `nats://${natsServer}:${natsPorts.server}`; -export const natsAccountName = "cocalc"; - -// I tested and if you make this bigger, then smaller, it does NOT break -// large jetstream messages created when it was bigger. So it should be -// safe to adjust. -// 1MB is the global NATS default -// const max_payload = "1MB"; -// Note that 64MB is the max allowed. -const max_payload = process.env.COCALC_NATS_MAX_PAYLOAD ?? "8MB"; -// However, using anything big means messages can take longer to send -// messages and risk timing out. I've also implemented chunking, -// *everywhere* it is needed. -// Clients do NOT cache the payload size so if you make it big, then make it -// small, that does not require restarting everything. - -export async function configureNatsServer() { - logger.debug("configureNatsServer", { confPath, natsPorts }); - if (await pathExists(confPath)) { - logger.debug( - `configureNatsServer: target conf file '${confPath}' already exists so updating it`, - ); - } - - let ISSUER_NKEY, ISSUER_XKEY, PASSWORD; - if (!natsPassword) { - PASSWORD = createPrivateKey("user"); - setNatsPassword(PASSWORD); - await writeFile(natsPasswordPath, PASSWORD); - } else { - PASSWORD = natsPassword; - } - if (!natsAuthCalloutNSeed) { - const nseed = createPrivateKey("account"); - setNatsAuthCalloutNSeed(nseed); - await writeFile(natsAuthCalloutNSeedPath, nseed); - ISSUER_NKEY = publicKey(nseed); - } else { - ISSUER_NKEY = publicKey(natsAuthCalloutNSeed); - } - if (!natsAuthCalloutXSeed) { - const xseed = createPrivateKey("curve"); - setNatsAuthCalloutXSeed(xseed); - await writeFile(natsAuthCalloutXSeedPath, xseed); - ISSUER_XKEY = publicKey(xseed); - } else { - ISSUER_XKEY = publicKey(natsAuthCalloutXSeed); - } - - // problem with server_name -- this line - // const user = fromPublic(userNkey); - // in server/nats/auth/index.ts fails. - - await writeFile( - confPath, - ` -# Amazingly, just setting the server_name breaks auth callout, -# with it saying the nkey is invalid. This may require a lot -# "reverse engineering" work. -# server_name: ${natsServerName} -listen: ${natsServer}:${natsPorts.server} - -max_payload:${max_payload} - -jetstream: enabled - -jetstream { - store_dir: data/nats/jetstream -} - -websocket { - listen: "${natsServer}:${natsPorts.ws}" - no_tls: true - token_cookie: "${REMEMBER_ME_COOKIE_NAME}" -} - -# This does not work yet. I guess a single node cluster -# isn't possible. Reload also isn't -- the only way we ever -# grow to multiple nodes will require restarts. -# cluster { -# name: "${natsClusterName}" -# listen: "${natsServer}:${natsPorts.cluster}" -# routes: ["${natsServer}:${natsPorts.cluster}"] -# compression: { -# mode: s2_auto -# } -# } - -accounts { - COCALC { - users: [ - { user:"${natsUser}", password:"${PASSWORD}" } - ], - jetstream: { - max_mem: -1 - max_file: -1 - max_streams: -1 - max_consumers: -1 - } - } - SYS { - users: [ - { user:"sys", password:"${PASSWORD}" } - ], - } -} -system_account: SYS - -max_control_line 64KB - -authorization { - # slightly longer timeout (than 2s default): probably not necessary, but db - # queries involved (usually takes 50ms - 250ms) - timeout: 7.5 - auth_callout { - issuer: ${ISSUER_NKEY} - xkey: ${ISSUER_XKEY} - users: [ ${natsUser}, sys ] - account: COCALC - } -} - -`, - ); - - // Ensure that ONLY we can read/write the nats config directory, - // which contains highly sensitive information. This could matter - // on cocalc-docker style systems. - await executeCode({ command: "chmod", args: ["og-rwx", nats] }); -} - -export async function main() { - await configureNatsServer(); - process.exit(0); -} diff --git a/src/packages/backend/nats/env.ts b/src/packages/backend/nats/env.ts deleted file mode 100644 index 598aac94b3..0000000000 --- a/src/packages/backend/nats/env.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { sha1 } from "@cocalc/backend/sha1"; -import { JSONCodec } from "nats"; -import { getConnection } from "./index"; - -export async function getEnv() { - const jc = JSONCodec(); - const nc = await getConnection(); - return { nc, jc, sha1 }; -} diff --git a/src/packages/backend/nats/index.ts b/src/packages/backend/nats/index.ts deleted file mode 100644 index cb2d80ea41..0000000000 --- a/src/packages/backend/nats/index.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { join } from "path"; -import { - nats, - natsPorts, - natsServer, - natsPassword, -} from "@cocalc/backend/data"; -import { readFile } from "node:fs/promises"; -import getLogger from "@cocalc/backend/logger"; -import { getEnv } from "./env"; -export { getEnv }; -import { inboxPrefix } from "@cocalc/nats/names"; -import { setNatsClient } from "@cocalc/nats/client"; -import getConnection, { - setConnectionOptions, -} from "@cocalc/backend/nats/persistent-connection"; -import { hostname } from "os"; - -export { getConnection }; - -export function init() { - setNatsClient({ getNatsEnv: getEnv, getLogger }); -} -init(); - -const logger = getLogger("backend:nats"); - -export async function getCreds(): Promise { - const filename = join(nats, "nsc/keys/creds/cocalc/cocalc/cocalc.creds"); - try { - return (await readFile(filename)).toString().trim(); - } catch { - logger.debug( - `getCreds -- please create ${filename}, which is missing. Nothing will work.`, - ); - return undefined; - } -} - -setConnectionOptions(async () => { - const servers = `${natsServer}:${natsPorts.server}`; - return { - user: "cocalc", - name: hostname(), - pass: natsPassword, - inboxPrefix: inboxPrefix({}), - servers, - }; -}); diff --git a/src/packages/backend/nats/install.ts b/src/packages/backend/nats/install.ts deleted file mode 100644 index 500ce77306..0000000000 --- a/src/packages/backend/nats/install.ts +++ /dev/null @@ -1,122 +0,0 @@ -/* -Ensure installed specific correct versions of the following -three GO programs in {data}/nats/bin on this server, correct -for this architecture: - - - nats - - nats-server - -We assume curl and python3 are installed. - -DEVELOPMENT: - -Installation happens automatically, e.g,. when you do 'pnpm nats-server' or -start the hub via 'pnpm hub'. However, you can explicitly do -an install as follows: - -~/cocalc/src/packages/backend/nats$ DEBUG=cocalc:* DEBUG_CONSOLE=yes node -Welcome to Node.js v18.17.1. -Type ".help" for more information. - -Install latest tested version of nats-server and nats cli: - - > await require('@cocalc/backend/nats/install').install() - -Installing just the server: - - > await require('@cocalc/backend/nats/install').installNatsServer() -*/ - -import { nats } from "@cocalc/backend/data"; -import { join } from "path"; -import { pathExists } from "fs-extra"; -import { executeCode } from "@cocalc/backend/execute-code"; -import getLogger from "@cocalc/backend/logger"; - -const VERSIONS = { - // https://github.com/nats-io/nats-server/releases - "nats-server": "v2.11.0", - // https://github.com/nats-io/natscli/releases - nats: "v0.2.0", -}; - -export const bin = join(nats, "bin"); -const logger = getLogger("backend:nats:install"); - -export async function install(noUpgrade = false) { - logger.debug("ensure nats binaries installed in ", bin); - - if (!(await pathExists(bin))) { - await executeCode({ command: "mkdir", args: ["-p", bin] }); - } - - await Promise.all([ - installNatsServer(noUpgrade), - installNatsCli(noUpgrade), - ]); -} - -// call often, but runs at most once and ONLY does something if -// there is no binary i.e., it doesn't upgrade. -let installed = false; -export async function ensureInstalled() { - if (installed) { - return; - } - installed = true; - await install(true); -} - -async function getVersion(name: string) { - try { - const { stdout } = await executeCode({ - command: join(bin, name), - args: ["--version"], - }); - const v = stdout.trim().split(/\s/g); - return v[v.length - 1]; - } catch { - return ""; - } -} - -export async function installNatsServer(noUpgrade) { - if (noUpgrade && (await pathExists(join(bin, "nats-server")))) { - return; - } - if ((await getVersion("nats-server")) == VERSIONS["nats-server"]) { - logger.debug( - `nats-server version ${VERSIONS["nats-server"]} already installed`, - ); - return; - } - const command = `curl -sf https://binaries.nats.dev/nats-io/nats-server/v2@${VERSIONS["nats-server"]} | sh`; - logger.debug("installing nats-server: ", command); - await executeCode({ - command, - path: bin, - verbose: true, - }); -} - -export async function installNatsCli(noUpgrade) { - if (noUpgrade && (await pathExists(join(bin, "nats")))) { - return; - } - if ((await getVersion("nats")) == VERSIONS["nats"]) { - logger.debug(`nats version ${VERSIONS["nats"]} already installed`); - return; - } - logger.debug("installing nats cli"); - await executeCode({ - command: `curl -sf https://binaries.nats.dev/nats-io/natscli/nats@${VERSIONS["nats"]} | sh`, - path: bin, - verbose: true, - }); -} - - -export async function main() { - await install(); - process.exit(0); -} diff --git a/src/packages/backend/nats/nkeys.ts b/src/packages/backend/nats/nkeys.ts deleted file mode 100644 index 7c6743fa52..0000000000 --- a/src/packages/backend/nats/nkeys.ts +++ /dev/null @@ -1,50 +0,0 @@ -/* -EXAMPLE: - -~/cocalc/src/packages/backend/nats$ n -Welcome to Node.js v18.17.1. -Type ".help" for more information. -> a = require('@cocalc/backend/nats/nkeys') -{ - publicKey: [Function: publicKey], - createPrivateKey: [Function: createPrivateKey] -} -> a.createPrivateKey('user') -'SUACDK5OBWPWYKHAZSKNO4IC3UXDYWD4LLPOVMM3DEY6Z7UXJQB3CK63B4' -> seed = a.createPrivateKey('user') -'SUACLFDTUS353H4ITLDAFQWYA43IAP2L7LGZ5XDEEARMJ4KNPHUWDKDUFQ' -> a.publicKey(seed) -'UCBWG2NENI2VLZRMXKAQOZVKVVPA5GBUY2G7KGEDJRDWFSQ5VV3P7VYD' -*/ - -import * as nkeys from "@nats-io/nkeys"; -import { capitalize } from "@cocalc/util/misc"; - -export function publicKey(seed: string): string { - const t = new TextEncoder(); - let kp; - if (seed.startsWith("SX")) { - kp = nkeys.fromCurveSeed(t.encode(seed)); - } else { - kp = nkeys.fromSeed(t.encode(seed)); - } - return kp.getPublicKey(); -} - -type KeyType = - | "account" - | "cluster" - | "curve" - | "operator" - | "pair" - | "server" - | "user"; - -export function createPrivateKey(type: KeyType): string { - const kp = nkeys[`create${capitalize(type)}`](); - const t = new TextDecoder(); - if (type == "curve") { - return t.decode(kp.getSeed()); - } - return t.decode(kp.seed); -} diff --git a/src/packages/backend/nats/persistent-connection.ts b/src/packages/backend/nats/persistent-connection.ts deleted file mode 100644 index 3cb5b62cca..0000000000 --- a/src/packages/backend/nats/persistent-connection.ts +++ /dev/null @@ -1,130 +0,0 @@ -/* -Create a nats connection that doesn't break. - -The NATS docs - -https://github.com/nats-io/nats.js/blob/main/core/README.md#connecting-to-a-nats-server - -ensure us that "the client will always attempt to reconnect if the connection is -disrupted for a reason other than calling close()" but THAT IS NOT TRUE. -(I think the upstream code in disconnected in nats.js/core/src/protocol.ts is a lazy -and I disagree with it. It tries to connect but if anything goes slightly wrong, -just gives up forever.) - -There are definitely situations where the connection gets permanently closed -and the close() function was not called, at least not by any of our code. -I've given up on getting them to fix or understand their bugs in general: - -https://github.com/williamstein/nats-bugs/issues/8 - -We thus monitor the connection, and if it closed, we *swap out the protocol -object*, which is an evil hack to reconnect. This seems to work fine with all -our other code. - -All that said, it's excellent that the NATS library separates the protocol from -the connection object itself, so it's possible to do this at all! :-) -*/ - -import { getLogger } from "@cocalc/backend/logger"; -import { delay } from "awaiting"; -import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; -import type { NatsConnection } from "@cocalc/nats/types"; -import { connect as connectViaTCP } from "nats"; -import { connect as connectViaWebsocket } from "nats.ws"; -import { CONNECT_OPTIONS } from "@cocalc/util/nats"; -import { WebSocket } from "ws"; - -const MONITOR_INTERVAL = 3000; - -const logger = getLogger("backend:nats:connection"); - -let options: any = null; -let getOptions: (() => Promise) | null = null; -export function setConnectionOptions(_getOptions: () => Promise) { - getOptions = _getOptions; -} - -let nc: NatsConnection | null = null; - -// gets the singleton connection -const getConnection = reuseInFlight(async (): Promise => { - if (nc == null) { - logger.debug("initializing nats cocalc backend connection"); - nc = await getNewConnection(); - monitorConnection(nc); - } - return nc; -}); - -export default getConnection; - -// NOTE: this monitorConnection also has to work properly with the -// waitUntilConnected function from @cocalc/nats/util. - -// The NATS docs ensure us that "the client will always attempt to -// reconnect if the connection is disrupted for a reason other than -// calling close()" but THAT IS NOT TRUE. There are many situations -// where the connection gets permanently closed and close was not -// called, at least not by any of our code. We thus monitor the -// connection, and if it closed, we *swap out the protocol object*, which -// is an evil hack to reconnect. This seems to work fine with all our -// other code. -async function monitorConnection(nc) { - while (true) { - if (nc.isClosed()) { - console.log("fixing the NATS connection..."); - const nc2 = await getNewConnection(); - // @ts-ignore - nc.protocol = nc2.protocol; - if (!nc.isClosed()) { - console.log("successfully fixed the NATS connection!"); - } else { - console.log("failed to fix the NATS connection!"); - } - } - await delay(MONITOR_INTERVAL); - } -} - -function getServer(servers) { - return typeof servers == "string" ? servers : servers[0]; -} - -export async function getNewConnection(): Promise { - logger.debug("create new connection"); - // make initial delay short, because secret token is being written to database - // right when project starts, so second attempt very likely to work. - let d = 1000; - while (true) { - try { - if (options == null && getOptions != null) { - options = { ...CONNECT_OPTIONS, ...(await getOptions()) }; - } - if (options == null) { - throw Error("options not set yet..."); - } - let connect; - if (getServer(options.servers).startsWith("ws")) { - // this is a workaround for a bug involving reconnect that I saw on some forum - // @ts-ignore - global.WebSocket = WebSocket; - connect = connectViaWebsocket; - } else { - connect = connectViaTCP; - } - logger.debug(`connecting to ${options.servers}`); - const conn = await connect({ ...options }); - if (conn == null) { - throw Error("connection failed"); - } - logger.debug(`connected to ${conn.getServer()}`); - return conn; - } catch (err) { - d = Math.min(15000, d * 1.35) + Math.random() / 2; - logger.debug( - `ERROR connecting to ${JSON.stringify(options?.servers)}; will retry in ${d / 1000} seconds. err=${err}`, - ); - await delay(d); - } - } -} diff --git a/src/packages/backend/nats/server.ts b/src/packages/backend/nats/server.ts deleted file mode 100644 index 2d657d50b3..0000000000 --- a/src/packages/backend/nats/server.ts +++ /dev/null @@ -1,46 +0,0 @@ -import { nats } from "@cocalc/backend/data"; -import { join } from "path"; -import { spawn, spawnSync } from "node:child_process"; - -function params() { - return { - command: join(nats, "bin", "nats-server"), - args: ["-c", join(nats, "server.conf")], - env: { cwd: nats }, - }; -} - -export function startServer(): number { - const { command, args, env } = params(); - const { pid } = spawn(command, args, env); - if (pid == null) { - throw Error("issue spawning nats-server"); - } - return pid; -} - -export function main({ - verbose, - daemon, -}: { verbose?: boolean; daemon?: boolean } = {}) { - let { command, args, env } = params(); - if (verbose) { - args = [...args, "-DV"]; - } - let opts; - if (daemon) { - opts = { ...env, detached: true, stdio: "ignore" }; - const child = spawn(command, args, opts); - child.on("error", (err) => { - throw Error(`Failed to start process: ${err}`); - }); - - if (daemon) { - console.log(`Process started as daemon with PID: ${child.pid}`); - child.unref(); - } - } else { - opts = { ...env, stdio: "inherit" }; - spawnSync(command, args, opts); - } -} diff --git a/src/packages/backend/nats/sync.ts b/src/packages/backend/nats/sync.ts deleted file mode 100644 index 2b2dd58348..0000000000 --- a/src/packages/backend/nats/sync.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { stream as createStream, type Stream } from "@cocalc/nats/sync/stream"; -import { - dstream as createDstream, - type DStream, -} from "@cocalc/nats/sync/dstream"; -import { kv as createKV, type KV } from "@cocalc/nats/sync/kv"; -import { dkv as createDKV, type DKV } from "@cocalc/nats/sync/dkv"; -import { dko as createDKO, type DKO } from "@cocalc/nats/sync/dko"; -import { akv as createAKV, type AKV } from "@cocalc/nats/sync/akv"; -import { createOpenFiles, type OpenFiles } from "@cocalc/nats/sync/open-files"; -export { inventory } from "@cocalc/nats/sync/inventory"; -import "./index"; - -export type { Stream, DStream, KV, DKV, DKO, AKV }; - -export async function stream(opts): Promise> { - return await createStream(opts); -} - -export async function dstream(opts): Promise> { - return await createDstream(opts); -} - -export async function kv(opts): Promise> { - return await createKV(opts); -} - -export async function dkv(opts): Promise> { - return await createDKV(opts); -} - -export function akv(opts): AKV { - return createAKV(opts); -} - -export async function dko(opts): Promise> { - return await createDKO(opts); -} - -export async function openFiles(project_id: string, opts?): Promise { - return await createOpenFiles({ project_id, ...opts }); -} diff --git a/src/packages/backend/nats/test/service.test.ts b/src/packages/backend/nats/test/service.test.ts deleted file mode 100644 index b216cbcae0..0000000000 --- a/src/packages/backend/nats/test/service.test.ts +++ /dev/null @@ -1,38 +0,0 @@ -/* - -DEVELOPMENT: - -pnpm test --forceExit service.test.ts - -*/ - -import { callNatsService, createNatsService } from "@cocalc/nats/service"; -import { once } from "@cocalc/util/async-utils"; -import "@cocalc/backend/nats"; - -describe("create a service and test it out", () => { - let s; - it("creates a service", async () => { - s = createNatsService({ - service: "echo", - handler: (mesg) => mesg.repeat(2), - }); - await once(s, "running"); - expect(await callNatsService({ service: "echo", mesg: "hello" })).toBe( - "hellohello", - ); - }); - - it("closes the services", async () => { - s.close(); - - let t = ""; - // expect( ...).toThrow doesn't seem to work with this: - try { - await callNatsService({ service: "echo", mesg: "hi", timeout: 1000 }); - } catch (err) { - t = `${err}`; - } - expect(t).toContain("Error: timeout"); - }); -}); diff --git a/src/packages/backend/nats/test/sync/binary.test.ts b/src/packages/backend/nats/test/sync/binary.test.ts deleted file mode 100644 index 6a20198448..0000000000 --- a/src/packages/backend/nats/test/sync/binary.test.ts +++ /dev/null @@ -1,116 +0,0 @@ -/* -Test using binary data with kv and stream. - -The default value type is json, which is heavily tested in all the other -unit tests. Here we test binary data instead. - -DEVELOPMENT: - -pnpm exec jest --forceExit "binary.test.ts" -*/ - -import "@cocalc/backend/nats"; // ensure client is setup -import { getMaxPayload } from "@cocalc/nats/util"; -import { dstream, dkv } from "@cocalc/backend/nats/sync"; - -describe("test binary data with a dstream", () => { - let s, - s2, - name = `${Math.random()}`; - - // binary values come back as Uint8Array with streams - const data10 = Uint8Array.from(Buffer.from("x".repeat(10))); - it("creates a binary dstream and writes/then reads binary data to/from it", async () => { - s = await dstream({ name, valueType: "binary" }); - expect(s.name).toBe(name); - s.publish(data10); - expect(s.get(0).length).toEqual(data10.length); - await s.close(); - s = await dstream({ name, valueType: "binary" }); - expect(s.get(0).length).toEqual(data10.length); - }); - - it("creates a dstream with the same name but json format and sees it is separate", async () => { - s2 = await dstream({ name, valueType: "json" }); - expect(s2.length).toBe(0); - s = await dstream({ name, valueType: "binary" }); - expect(s.length).toBe(1); - s2.push({ hello: "cocalc" }); - expect(s.length).toBe(1); - expect(s2.length).toBe(1); - await s2.close(); - s2 = await dstream({ name, valueType: "json" }); - expect(s2.get(0)).toEqual({ hello: "cocalc" }); - }); - - it("writes large binary data to the dstream to test chunking", async () => { - s = await dstream({ name, valueType: "binary" }); - const maxPayload = await getMaxPayload(); - const data = Uint8Array.from(Buffer.from("x".repeat(maxPayload * 1.5))); - s.publish(data); - expect(s.get(s.length - 1).length).toEqual(data.length); - await s.close(); - s = await dstream({ name, valueType: "binary" }); - expect(s.get(s.length - 1).length).toEqual(data.length); - }); - - it("clean up", async () => { - await s.purge(); - await s.close(); - await s2.purge(); - await s2.close(); - }); -}); - -describe("test binary data with a dkv", () => { - let s, - name = `${Math.random()}`; - - // binary values come back as buffer with dkv - const data10 = Buffer.from("x".repeat(10)); - - it("creates a binary dkv and writes/then reads binary data to/from it", async () => { - s = await dkv({ name, valueType: "binary" }); - expect(s.name).toBe(name); - s.x = data10; - expect(s.x).toEqual(data10); - expect(s.x.length).toEqual(data10.length); - await s.close(); - s = await dkv({ name, valueType: "binary" }); - expect(s.x.length).toEqual(data10.length); - expect(s.x).toEqual(data10); - }); - - let s2; - it("creates a dkv with the same name but json format and sees it is separate", async () => { - s2 = await dkv({ name, valueType: "json" }); - expect(s2.length).toBe(0); - s = await dkv({ name, valueType: "binary" }); - expect(s.length).toBe(1); - s2.x = { hello: "cocalc" }; - expect(s.length).toBe(1); - expect(s2.length).toBe(1); - await s2.close(); - s2 = await dkv({ name, valueType: "json" }); - expect(s2.x).toEqual({ hello: "cocalc" }); - expect(s.x.length).toEqual(data10.length); - }); - - it("writes large binary data to the dkv to test chunking", async () => { - s = await dkv({ name, valueType: "binary" }); - const maxPayload = await getMaxPayload(); - const data = Uint8Array.from(Buffer.from("x".repeat(maxPayload * 1.5))); - s.y = data; - expect(s.y.length).toEqual(data.length); - await s.close(); - s = await dkv({ name, valueType: "binary" }); - expect(s.y.length).toEqual(data.length); - }); - - it("clean up", async () => { - await s.clear(); - await s.close(); - await s2.clear(); - await s2.close(); - }); -}); diff --git a/src/packages/backend/nats/test/sync/chunk.test.ts b/src/packages/backend/nats/test/sync/chunk.test.ts deleted file mode 100644 index bf0cf835d2..0000000000 --- a/src/packages/backend/nats/test/sync/chunk.test.ts +++ /dev/null @@ -1,67 +0,0 @@ -/* -We support arbitrarily large values for both our kv store and stream. - -This tests that this actually works. - -DEVELOPMENT: - -pnpm exec jest --forceExit "chunk.test.ts" - -WARNING: - -If this suddenly breaks, see the comment in packages/nats/sync/general-kv.ts -about potentially having to fork NATS. -*/ - -import "@cocalc/backend/nats"; // ensure client is setup -import { getMaxPayload } from "@cocalc/nats/util"; -import { createDstream } from "./util"; -import { dstream } from "@cocalc/backend/nats/sync"; -import { dkv as createDkv } from "@cocalc/backend/nats/sync"; - -describe("create a dstream and a dkv and write a large chunk to each", () => { - let maxPayload = 0; - - it("sanity check on the max payload", async () => { - maxPayload = await getMaxPayload(); - expect(maxPayload).toBeGreaterThan(1000000); - }); - - it("write a large value with a dstream", async () => { - const largeValue = "x".repeat(2.5 * maxPayload); - const stream = await createDstream(); - stream.push(largeValue); - expect(stream[0].length).toBe(largeValue.length); - expect(stream[0] == largeValue).toBe(true); - await stream.save(); - expect(stream.hasUnsavedChanges()).toBe(false); - const name = stream.name; - await stream.close(); - - const stream2 = await dstream({ name, noAutosave: true }); - expect(stream2[0].length).toBe(largeValue.length); - expect(stream2[0] == largeValue).toBe(true); - // @ts-ignore some modicum of cleanup... - await stream2.stream.purge(); - }); - - it("write a large value to a dkv", async () => { - const name = `test-${Math.random()}`; - const largeValue = "x".repeat(2.5 * maxPayload); - const dkv = await createDkv({ name }); - dkv.set("a", largeValue); - expect(dkv.get("a").length).toBe(largeValue.length); - await dkv.save(); - expect(dkv.hasUnsavedChanges()).toBe(false); - await dkv.close(); - - const dkv2 = await createDkv({ name, noAutosave: true }); - expect(dkv2.get("a").length).toBe(largeValue.length); - expect(dkv2.get("a") == largeValue).toBe(true); - // @ts-ignore some modicum of cleanup... - await dkv2.delete("a"); - }); -}); - -// TODO: the above is just the most minimal possible test. a million things -// aren't tested yet... diff --git a/src/packages/backend/nats/test/sync/general-kv.test.ts b/src/packages/backend/nats/test/sync/general-kv.test.ts deleted file mode 100644 index 2f420adee4..0000000000 --- a/src/packages/backend/nats/test/sync/general-kv.test.ts +++ /dev/null @@ -1,235 +0,0 @@ -/* -A lot of GeneralKV is indirectly unit tested because many other things -build on it, and they are tested, e.g., dkv. But it's certainly good -to test the basics here directly as well, since if something goes wrong, -it'll be easier to track down with lower level tests in place. - -DEVELOPMENT: - -pnpm exec jest --forceExit "general-kv.test.ts" - -*/ -// import { once } from "@cocalc/util/async-utils"; -import { delay } from "awaiting"; -import { getEnv } from "@cocalc/backend/nats/env"; -import { GeneralKV } from "@cocalc/nats/sync/general-kv"; -import { getMaxPayload } from "@cocalc/nats/util"; - -describe("create a general kv and do basic operations", () => { - let kv, kv2, kv3, env; - const name = `test-${Math.round(1000 * Math.random())}`; - - it("creates the kv", async () => { - env = await getEnv(); - kv = new GeneralKV({ name, env, filter: ["foo.>"] }); - await kv.init(); - await kv.clear(); - }); - - it("sets and deletes a key", async () => { - await kv.set("foo.x", 10); - expect(kv.getAll()).toEqual({ "foo.x": 10 }); - await kv.delete("foo.x"); - expect(kv.getAll()).toEqual({}); - await kv.set("foo.x", 10); - }); - - it("a second kv with a different filter", async () => { - kv2 = new GeneralKV({ name, env, filter: ["bar.>"] }); - await kv2.init(); - await kv2.clear(); - expect(kv2.getAll()).toEqual({}); - await kv2.set("bar.abc", 10); - expect(await kv2.getAll()).toEqual({ "bar.abc": 10 }); - expect(kv.getAll()).toEqual({ "foo.x": 10 }); - }); - - it("the union", async () => { - kv3 = new GeneralKV({ name, env, filter: ["bar.>", "foo.>"] }); - await kv3.init(); - expect(kv3.getAll()).toEqual({ "foo.x": 10, "bar.abc": 10 }); - }); - - it("clear and closes the kv", async () => { - await kv.clear(); - kv.close(); - await kv2.clear(); - kv2.close(); - }); -}); - -// NOTE: with these tests, we're "dancing" with https://github.com/nats-io/nats.js/issues/246 -// and might be forced to fork nats.js. Let's hope not! -describe("test that complicated keys work", () => { - let kv, env; - const name = `test-${Math.round(1000 * Math.random())}`; - - it("creates the kv", async () => { - env = await getEnv(); - kv = new GeneralKV({ name, env, filter: ["foo.>"] }); - await kv.init(); - }); - - it("creates complicated keys that ARE allowed", async () => { - for (const k of [ - `foo.${base64}`, - "foo.!@#$%^&()", - "foo.bar.baz!.bl__-+#@ah.nat\\s", - "foo.CoCalc-和-NATS-的结合非常棒!", - // and a VERY long 50kb key: - "foo." + "x".repeat(50000), - ]) { - await kv.set(k, "cocalc"); - expect(kv.get(k)).toEqual("cocalc"); - } - }); - - it("creates keys that are NOT allowed", async () => { - for (const k of [ - "foo.b c", - "foo.", - "foo.bar.", - "foo.b\u0000c", - "foo.b*c", - "foo.b>c", - ]) { - expect(async () => await kv.set(k, "not-allowed")).rejects.toThrow(); - } - }); - - it("clear and closes the kv", async () => { - await kv.clear(); - kv.close(); - }); -}); - -const base64 = - "0123456789+/ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz="; - -describe("a complicated filter", () => { - let kv, env; - const name = `test-${Math.round(1000 * Math.random())}`; - - it("creates the kv", async () => { - env = await getEnv(); - kv = new GeneralKV({ name, env, filter: [`${base64}.>`] }); - await kv.init(); - }); - it("clear and closes the kv", async () => { - await kv.clear(); - kv.close(); - }); -}); - -describe("test using the binary value type", () => { - let kv, env; - const name = `test-${Math.round(1000 * Math.random())}`; - - it("creates the kv", async () => { - env = await getEnv(); - kv = new GeneralKV({ name, env, filter: ["foo.>"], valueType: "binary" }); - await kv.init(); - }); - - it("set and get a binary value", async () => { - const value = Buffer.from([0, 0, 3, 8, 9, 5, 0, 7, 7]); - await kv.set("foo.b", value); - expect(kv.get("foo.b")).toEqual(value); - expect(kv.get("foo.b").length).toEqual(9); - }); - - it("sets and gets a large binary value that requires chunking", async () => { - const m = await getMaxPayload(); - const value = Buffer.from("x".repeat(1.5 * m)); - value[0] = 7; - await kv.set("foo.big", value); - expect(kv.get("foo.big").length).toEqual(value.length); - }); - - it("clear and closes the kv", async () => { - await kv.clear(); - kv.close(); - }); -}); - -describe("test using a range of useful functions: length, has, time, headers, etc.", () => { - let kv, env; - const name = `test-${Math.round(1000 * Math.random())}`; - - it("creates the kv", async () => { - env = await getEnv(); - kv = new GeneralKV({ name, env, filter: ["foo.>"] }); - await kv.init(); - }); - - it("sets a value and observe length matches", async () => { - expect(kv.length).toBe(0); - await kv.set("foo.x", 10); - expect(kv.length).toBe(1); - }); - - it("sets a value and observe time is reasonable", async () => { - await kv.set("foo.time", 10); - while (kv.time("foo.time") == null) { - await delay(10); - } - expect(Math.abs(kv.time("foo.time").valueOf() - Date.now())).toBeLessThan( - 10000, - ); - }); - - it("check has works", async () => { - expect(await kv.has("foo.has")).toBe(false); - await kv.set("foo.has", "it"); - expect(await kv.has("foo.has")).toBe(true); - await kv.delete("foo.has"); - expect(await kv.has("foo.has")).toBe(false); - }); - - it("verifying key is valid given the filter", async () => { - expect(kv.isValidKey("foo.x")).toBe(true); - expect(kv.isValidKey("bar.x")).toBe(false); - }); - - it("expire keys using ageMs", async () => { - await kv.set("foo.old", 10); - await delay(100); - await kv.set("foo.new", 20); - await kv.expire({ ageMs: 200 }); - expect(kv.has("foo.old")).toBe(true); - await kv.expire({ ageMs: 50 }); - expect(kv.has("foo.old")).toBe(false); - expect(kv.has("foo.new")).toBe(true); - }); - - it("expire keys using cutoff", async () => { - await kv.set("foo.old0", 10); - await delay(50); - const cutoff = new Date(); - await delay(50); - await kv.set("foo.new0", 20); - await kv.expire({ cutoff }); - expect(kv.has("foo.old0")).toBe(false); - expect(kv.has("foo.new0")).toBe(true); - }); - - it("sets and gets a header", async () => { - await kv.set("foo.head", 10, { headers: { CoCalc: "NATS" } }); - expect(kv.get("foo.head")).toBe(10); - while (kv.headers("foo.head") == null) { - await delay(10); - } - expect(kv.headers("foo.head").CoCalc).toBe("NATS"); - }); - - it("sanity check on stats", async () => { - const stats = kv.stats(); - expect(stats.count).toBeGreaterThan(0); - expect(stats.bytes).toBeGreaterThan(0); - }); - - it("clear and closes the kv", async () => { - await kv.clear(); - kv.close(); - }); -}); diff --git a/src/packages/backend/nats/test/sync/headers.test.ts b/src/packages/backend/nats/test/sync/headers.test.ts deleted file mode 100644 index 4211287387..0000000000 --- a/src/packages/backend/nats/test/sync/headers.test.ts +++ /dev/null @@ -1,170 +0,0 @@ -/* -Test using user-defined headers with kv and stream. - -DEVELOPMENT: - -pnpm exec jest --forceExit "headers.test.ts" -*/ - -import "@cocalc/backend/nats"; // ensure client is setup -import { getMaxPayload } from "@cocalc/nats/util"; -import { dstream, stream, dkv, kv } from "@cocalc/backend/nats/sync"; -import { once } from "@cocalc/util/async-utils"; - -describe("test headers with a stream", () => { - let s; - it("creates a stream and writes a value without a header", async () => { - s = await stream({ name: `${Math.random()}` }); - expect(s.headers(s.length - 1)).toBe(undefined); - s.publish("x"); - await once(s, "change"); - expect(s.headers(s.length - 1)).toBe(undefined); - }); - - it("writes a value with a header", async () => { - s.publish("y", { headers: { my: "header" } }); - await once(s, "change"); - expect(s.headers(s.length - 1)).toEqual({ my: "header" }); - }); - - it("writes a large value to a stream that requires chunking and a header", async () => { - s.publish("y".repeat((await getMaxPayload()) * 2), { - headers: { large: "chunks", multiple: "keys" }, - }); - await once(s, "change"); - expect(s.headers(s.length - 1)).toEqual( - expect.objectContaining({ large: "chunks", multiple: "keys" }), - ); - expect(s.headers(s.length - 1)).toEqual({ - large: "chunks", - multiple: "keys", - // CoCalc- and Nats- headers get used internally, but are still visible. - // 3 because of how size was chosen above. - "CoCalc-Chunks": "3/3", - }); - }); - - it("clean up", async () => { - await s.purge(); - }); -}); - -describe("test headers with a dstream", () => { - let s; - const name = `${Math.random()}`; - it("creates a dstream and writes a value without a header", async () => { - s = await dstream({ name }); - expect(s.headers(s.length - 1)).toBe(undefined); - s.publish("x"); - await once(s, "change"); - const h = s.headers(s.length - 1); - for (const k in h ?? {}) { - if (!k.startsWith("Nats-") && !k.startsWith("CoCalc-")) { - throw Error("headers must start with Nats- or CoCalc-"); - } - } - }); - - it("writes a value with a header", async () => { - s.publish("y", { headers: { my: "header" } }); - // NOTE: not optimal but this is what is implemented and documented! - expect(s.headers(s.length - 1)).toEqual(undefined); - await once(s, "change"); - expect(s.headers(s.length - 1)).toEqual( - expect.objectContaining({ my: "header" }), - ); - }); - - it("header still there", async () => { - await s.close(); - s = await dstream({ name }); - expect(s.headers(s.length - 1)).toEqual( - expect.objectContaining({ my: "header" }), - ); - }); - - it("clean up", async () => { - await s.purge(); - }); -}); - -describe("test headers with low level general kv", () => { - let s, gkv; - it("creates a kv and writes a value without a header", async () => { - s = await kv({ name: `${Math.random()}` }); - gkv = s.generalKV; - const key = `${s.prefix}.x`; - expect(gkv.headers(key)).toBe(undefined); - gkv.set(key, 10); - await once(gkv, "change"); - expect(gkv.headers(key)).toBe(undefined); - }); - - it("writes a value with a header", async () => { - const key = `${s.prefix}.y`; - gkv.set(key, 20, { headers: { my: "header" } }); - await once(gkv, "change"); - expect(gkv.headers(key)).toEqual({ my: "header" }); - }); - - it("changes header without changing value", async () => { - const key = `${s.prefix}.y`; - gkv.set(key, 20, { headers: { my: "header2", second: "header" } }); - await once(gkv, "change"); - expect(gkv.headers(key)).toEqual( - expect.objectContaining({ my: "header2", second: "header" }), - ); - }); - - it("removes header without changing value", async () => { - const key = `${s.prefix}.y`; - gkv.set(key, 20, { headers: { my: null, second: "header" } }); - await once(gkv, "change"); - expect(gkv.headers(key)).toEqual( - expect.objectContaining({ second: "header" }), - ); - }); - - it("writes a large value to a kv that requires chunking and a header", async () => { - const key = `${s.prefix}.big`; - gkv.set(key, "x".repeat((await getMaxPayload()) * 2), { - headers: { the: "header" }, - }); - await once(gkv, "change"); - expect(gkv.headers(key)).toEqual( - expect.objectContaining({ the: "header" }), - ); - }); - - it("clean up", async () => { - await s.clear(); - }); -}); - -describe("test headers with a dkv", () => { - let s; - const name = `${Math.random()}`; - it("creates a dkv and writes a value without a header", async () => { - s = await dkv({ name }); - s.set("x", 10); - await once(s, "change"); - const h = s.headers("x"); - for (const k in h ?? {}) { - if (!k.startsWith("Nats-") && !k.startsWith("CoCalc-")) { - throw Error("headers must start with Nats- or CoCalc-"); - } - } - }); - - it("writes a value with a header", async () => { - s.set("y", 20, { headers: { my: "header" } }); - // NOTE: not optimal but this is what is implemented and documented! - expect(s.headers("y")).toEqual(undefined); - await once(s, "change"); - expect(s.headers("y")).toEqual(expect.objectContaining({ my: "header" })); - }); - - it("clean up", async () => { - await s.clear(); - }); -}); diff --git a/src/packages/backend/nats/test/sync/limits.test.ts b/src/packages/backend/nats/test/sync/limits.test.ts deleted file mode 100644 index 5f5d4a5751..0000000000 --- a/src/packages/backend/nats/test/sync/limits.test.ts +++ /dev/null @@ -1,259 +0,0 @@ -/* -Testing the limits. - -DEVELOPMENT: - -pnpm exec jest --forceExit "limits.test.ts" - -*/ - -import { dkv as createDkv } from "@cocalc/backend/nats/sync"; -import { dstream as createDstream } from "@cocalc/backend/nats/sync"; -import { delay } from "awaiting"; -import { once } from "@cocalc/util/async-utils"; - -describe.skip("create a dkv with limit on the total number of keys, and confirm auto-delete works", () => { - let kv; - const name = `test-${Math.random()}`; - - it("creates the dkv", async () => { - kv = await createDkv({ name, limits: { max_msgs: 2 } }); - expect(kv.getAll()).toEqual({}); - }); - - it("adds 2 keys, then a third, and sees first is gone", async () => { - kv.a = 10; - kv.b = 20; - expect(kv.a).toEqual(10); - expect(kv.b).toEqual(20); - kv.c = 30; - expect(kv.c).toEqual(30); - // have to wait until it's all saved and acknowledged before enforcing limit - if (!kv.isStable()) { - await once(kv, "stable"); - } - // cause limit enforcement immediately so unit tests aren't slow - await kv.generalDKV.kv.enforceLimitsNow(); - // next change is the enforcement happening - if (kv.has("a")) { - await once(kv, "change", 500); - } - // and confirm it - expect(kv.a).toBe(undefined); - expect(kv.getAll()).toEqual({ b: 20, c: 30 }); - }); - - it("closes the kv", async () => { - await kv.clear(); - await kv.close(); - }); -}); - -describe.skip("create a dkv with limit on age of keys, and confirm auto-delete works", () => { - let kv; - const name = `test-${Math.random()}`; - - it("creates the dkv", async () => { - kv = await createDkv({ name, limits: { max_age: 50 } }); - expect(kv.getAll()).toEqual({}); - }); - - it("adds 2 keys, then a third, and sees first two are gone due to aging out", async () => { - kv.a = 10; - kv.b = 20; - expect(kv.a).toEqual(10); - expect(kv.b).toEqual(20); - await kv.save(); - await delay(75); - kv.c = 30; - expect(kv.c).toEqual(30); - if (!kv.isStable()) { - await once(kv, "stable"); - } - await kv.generalDKV.kv.enforceLimitsNow(); - if (kv.has("a")) { - await once(kv, "change", 500); - } - expect(kv.getAll()).toEqual({ c: 30 }); - }); - - it("closes the kv", async () => { - await kv.clear(); - await kv.close(); - }); -}); - -describe("create a dkv with limit on total bytes of keys, and confirm auto-delete works", () => { - let kv; - const name = `test-${Math.random()}`; - - it("creates the dkv", async () => { - kv = await createDkv({ name, limits: { max_bytes: 100 } }); - expect(kv.getAll()).toEqual({}); - }); - - it("adds a key, then a seocnd, and sees first one is gone due to bytes", async () => { - kv.a = "x".repeat(50); - await kv.save(); - kv.b = "x".repeat(75); - if (!kv.isStable()) { - await once(kv, "stable"); - } - await delay(250); - await kv.generalDKV.kv.enforceLimitsNow(); - if (kv.has("a")) { - await once(kv, "change", 500); - } - expect(kv.getAll()).toEqual({ b: "x".repeat(75) }); - }); - - it("closes the kv", async () => { - await kv.clear(); - await kv.close(); - }); -}); - -describe.skip("create a dkv with limit on max_msg_size, and confirm writing small messages works but writing a big one result in a 'reject' event", () => { - let kv; - const name = `test-${Math.random()}`; - - it("creates the dkv", async () => { - kv = await createDkv({ name, limits: { max_msg_size: 100 } }); - expect(kv.getAll()).toEqual({}); - }); - - it("adds a key, then a second big one results in a 'reject' event", async () => { - const rejects: { key: string; value: string }[] = []; - kv.once("reject", (x) => { - rejects.push(x); - }); - kv.a = "x".repeat(50); - await kv.save(); - kv.b = "x".repeat(150); - await kv.save(); - expect(rejects).toEqual([{ key: "b", value: "x".repeat(150) }]); - expect(kv.has("b")).toBe(false); - }); - - it("closes the kv", async () => { - await kv.clear(); - await kv.close(); - }); -}); - -describe("create a dstream with limit on the total number of messages, and confirm auto-delete works", () => { - let s; - const name = `test-${Math.random()}`; - - it("creates the dstream", async () => { - s = await createDstream({ name, limits: { max_msgs: 2 } }); - expect(s.get()).toEqual([]); - }); - - it("push 2 messages, then a third, and sees first is gone", async () => { - s.push({ a: 10 }); - s.push({ b: 20 }); - expect(s.get()).toEqual([{ a: 10 }, { b: 20 }]); - s.push({ c: 30 }); - expect(s.get(2)).toEqual({ c: 30 }); - // have to wait until it's all saved and acknowledged before enforcing limit - if (!s.isStable()) { - await once(s, "stable"); - } - // cause limit enforcement immediately so unit tests aren't slow - await s.stream.enforceLimitsNow(); - expect(s.getAll()).toEqual([{ b: 20 }, { c: 30 }]); - - // also check limits was enforced if we close, then open new one: - await s.close(); - s = await createDstream({ name, limits: { max_msgs: 2 } }); - expect(s.getAll()).toEqual([{ b: 20 }, { c: 30 }]); - }); - - it("closes the stream", async () => { - await s.purge(); - await s.close(); - }); -}); - -describe("create a dstream with limit on max_age, and confirm auto-delete works", () => { - let s; - const name = `test-${Math.random()}`; - - it("creates the dstream", async () => { - s = await createDstream({ name, limits: { max_age: 50 } }); - }); - - it("push a message, then another and see first disappears", async () => { - s.push({ a: 10 }); - await delay(100); - s.push({ b: 20 }); - expect(s.get()).toEqual([{ a: 10 }, { b: 20 }]); - if (!s.isStable()) { - await once(s, "stable"); - } - await s.stream.enforceLimitsNow(); - expect(s.getAll()).toEqual([{ b: 20 }]); - }); - - it("closes the stream", async () => { - await s.purge(); - await s.close(); - }); -}); - -describe("create a dstream with limit on max_bytes, and confirm auto-delete works", () => { - let s; - const name = `test-${Math.random()}`; - - it("creates the dstream", async () => { - s = await createDstream({ name, limits: { max_bytes: 50 } }); - }); - - it("push a message, then another and see first disappears", async () => { - s.push("x".repeat(40)); - s.push("x".repeat(45)); - s.push("x"); - if (!s.isStable()) { - await once(s, "stable"); - } - await s.stream.enforceLimitsNow(); - expect(s.getAll()).toEqual(["x".repeat(45), "x"]); - }); - - it("closes the stream", async () => { - await s.purge(); - await s.close(); - }); -}); - -describe("create a dstream with limit on max_msg_size, and confirm auto-delete works", () => { - let s; - const name = `test-${Math.random()}`; - - it("creates the dstream", async () => { - s = await createDstream({ name, limits: { max_msg_size: 50 } }); - }); - - it("push a message, then another and see first disappears", async () => { - const rejects: any[] = []; - s.on("reject", ({ mesg }) => { - rejects.push(mesg); - }); - s.push("x".repeat(40)); - s.push("y".repeat(60)); // silently vanishes (well a reject event is emitted) - s.push("x"); - if (!s.isStable()) { - await once(s, "stable"); - } - await s.stream.enforceLimitsNow(); - expect(s.getAll()).toEqual(["x".repeat(40), "x"]); - - expect(rejects).toEqual(["y".repeat(60)]); - }); - - it("closes the stream", async () => { - await s.purge(); - await s.close(); - }); -}); diff --git a/src/packages/backend/package.json b/src/packages/backend/package.json index 9b6459ce34..a452132587 100644 --- a/src/packages/backend/package.json +++ b/src/packages/backend/package.json @@ -5,55 +5,48 @@ "exports": { "./*": "./dist/*.js", "./database": "./dist/database/index.js", - "./nats": "./dist/nats/index.js", + "./conat": "./dist/conat/index.js", "./server-settings": "./dist/server-settings/index.js", "./auth/*": "./dist/auth/*.js", "./auth/tokens/*": "./dist/auth/tokens/*.js" }, - "keywords": [ - "utilities", - "cocalc" - ], + "keywords": ["utilities", "cocalc"], "scripts": { "preinstall": "npx only-allow pnpm", "clean": "rm -rf dist node_modules", "build": "pnpm exec tsc --build", "tsc": "pnpm exec tsc --watch --pretty --preserveWatchOutput", - "test": "pnpm exec jest --forceExit --runInBand", - "prepublishOnly": "pnpm test" + "test": "pnpm exec jest --forceExit --maxWorkers=50%", + "depcheck": "pnpx depcheck --ignores events", + "prepublishOnly": "pnpm test", + "conat-watch": "node ./bin/conat-watch.cjs", + "conat-connections": "node ./bin/conat-connections.cjs", + "conat-disconnect": "node ./bin/conat-disconnect.cjs", + "conat-inventory": "node ./bin/conat-inventory.cjs", + "conat-persist": "DEBUG=cocalc:* node ./bin/conat-persist.cjs" }, - "files": [ - "dist/**", - "bin/**", - "README.md", - "package.json" - ], + "files": ["dist/**", "bin/**", "README.md", "package.json"], "author": "SageMath, Inc.", "license": "SEE LICENSE.md", "dependencies": { "@cocalc/backend": "workspace:*", - "@cocalc/nats": "workspace:*", + "@cocalc/conat": "workspace:*", "@cocalc/util": "workspace:*", - "@nats-io/nkeys": "^2.0.3", "@types/debug": "^4.1.12", - "@types/watchpack": "^2.4.4", - "@types/ws": "^8.18.1", + "@types/jest": "^29.5.14", "awaiting": "^3.0.0", + "better-sqlite3": "^11.10.0", "chokidar": "^3.6.0", "debug": "^4.4.0", "fs-extra": "^11.2.0", "lodash": "^4.17.21", "lru-cache": "^7.18.3", - "nats": "^2.29.3", - "nats.ws": "^1.30.2", "password-hash": "^1.2.2", "prom-client": "^13.0.0", "rimraf": "^5.0.5", "shell-escape": "^0.2.0", - "supports-color": "^9.0.2", "tmp-promise": "^3.0.3", - "underscore": "^1.12.1", - "ws": "^8.18.0" + "zstd-napi": "^0.0.10" }, "repository": { "type": "git", @@ -61,7 +54,6 @@ }, "homepage": "https://github.com/sagemathinc/cocalc/tree/master/src/packages/backend", "devDependencies": { - "@types/node": "^18.16.14", - "expect": "^26.6.2" + "@types/node": "^18.16.14" } } diff --git a/src/packages/backend/tsconfig.json b/src/packages/backend/tsconfig.json index 8d855cf7c6..148fecfd62 100644 --- a/src/packages/backend/tsconfig.json +++ b/src/packages/backend/tsconfig.json @@ -7,5 +7,5 @@ "outDir": "dist" }, "exclude": ["node_modules", "dist", "test"], - "references": [{ "path": "../util", "path": "../nats" }] + "references": [{ "path": "../util", "path": "../conat" }] } diff --git a/src/packages/comm/package.json b/src/packages/comm/package.json index f468e526b1..813f6b1cbe 100644 --- a/src/packages/comm/package.json +++ b/src/packages/comm/package.json @@ -8,27 +8,33 @@ "./project-status/*": "./dist/project-status/*.js", "./project-info/*": "./dist/project-info/*.js" }, - "files": ["dist/**", "README.md", "package.json", "tsconfig.json"], + "files": [ + "dist/**", + "README.md", + "package.json", + "tsconfig.json" + ], "scripts": { "preinstall": "npx only-allow pnpm", "build": "../node_modules/.bin/tsc --build", - "tsc": "../node_modules/.bin/tsc --watch --pretty --preserveWatchOutput" + "tsc": "../node_modules/.bin/tsc --watch --pretty --preserveWatchOutput", + "depcheck": "pnpx depcheck --ignores @types/node" }, "author": "SageMath, Inc.", - "keywords": ["cocalc"], + "keywords": [ + "cocalc" + ], "license": "SEE LICENSE.md", - "dependencies": {}, - "homepage": "https://github.com/sagemathinc/cocalc/tree/master/src/packages/comm", - "repository": { - "type": "git", - "url": "https://github.com/sagemathinc/cocalc" - }, "dependencies": { "@cocalc/comm": "workspace:*", - "@cocalc/jupyter": "workspace:*", "@cocalc/sync": "workspace:*", "@cocalc/util": "workspace:*" }, + "homepage": "https://github.com/sagemathinc/cocalc/tree/master/src/packages/comm", + "repository": { + "type": "git", + "url": "https://github.com/sagemathinc/cocalc" + }, "devDependencies": { "@types/node": "^18.16.14" } diff --git a/src/packages/comm/project-configuration.ts b/src/packages/comm/project-configuration.ts index 7e78735513..37a579d88e 100644 --- a/src/packages/comm/project-configuration.ts +++ b/src/packages/comm/project-configuration.ts @@ -10,7 +10,7 @@ export const LIBRARY_INDEX_FILE = "/ext/library/cocalc-examples/index.json"; export interface MainConfiguration { capabilities: MainCapabilities; - timestamp: string; + timestamp: Date; // disabled extensions, for opening/creating files disabled_ext: string[]; } @@ -18,7 +18,7 @@ export interface MainConfiguration { export type Capabilities = { [key: string]: boolean }; export interface X11Configuration { - timestamp: string; + timestamp: Date; capabilities: Capabilities; } diff --git a/src/packages/comm/websocket/types.ts b/src/packages/comm/websocket/types.ts index 4264656c5a..40be8e8af4 100644 --- a/src/packages/comm/websocket/types.ts +++ b/src/packages/comm/websocket/types.ts @@ -142,23 +142,6 @@ interface MesgX11Channel { display: number; } -interface MesgSynctableChannel { - cmd: "synctable_channel"; - query: any; - options: any[]; -} - -interface MesgSyncdocCall { - cmd: "syncdoc_call"; - path: string; - mesg: any; -} - -interface MesgSymmetricChannel { - cmd: "symmetric_channel"; - name: string; -} - interface MesgRealpath { cmd: "realpath"; path: string; @@ -254,9 +237,6 @@ export type Mesg = | MesgLeanChannel | MesgQuery | MesgX11Channel - | MesgSynctableChannel - | MesgSyncdocCall - | MesgSymmetricChannel | MesgRealpath | MesgNBGrader | MesgJupyterNbconvert diff --git a/src/packages/nats/client.ts b/src/packages/conat/client.ts similarity index 55% rename from src/packages/nats/client.ts rename to src/packages/conat/client.ts index c706c79e77..5dc2148d17 100644 --- a/src/packages/nats/client.ts +++ b/src/packages/conat/client.ts @@ -2,21 +2,17 @@ DEVELOPMENT: ~/cocalc/src/packages/backend$ node -> require('@cocalc/backend/nats'); c = require('@cocalc/nats/client').getClient() +> require('@cocalc/backend/conat'); c = require('@cocalc/conat/client').getClient() > c.state 'connected' -> Object.keys(await c.getNatsEnv()) -[ 'nc', 'jc' ] */ -import type { NatsEnv, NatsEnvFunction } from "@cocalc/nats/types"; import { init } from "./time"; import { EventEmitter } from "events"; -import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; -import type { NatsConnection } from "@nats-io/nats-core"; +import { type Client as ConatClient } from "@cocalc/conat/core/client"; interface Client { - getNatsEnv: NatsEnvFunction; + conat: (opts?) => ConatClient; account_id?: string; project_id?: string; compute_server_id?: number; @@ -37,34 +33,34 @@ const FALLBACK_LOGGER = { debug: () => {}, info: () => {}, warn: () => {}, + silly: () => {}, } as Logger; export class ClientWithState extends EventEmitter { - getNatsEnv: NatsEnvFunction; + conatClient?: ConatClient; account_id?: string; project_id?: string; compute_server_id?: number; state: State = "disconnected"; - env?: NatsEnv; _getLogger?: (name) => Logger; _reconnect?: () => Promise; + conat: () => ConatClient; constructor(client: Client) { super(); - // many things listen for these events -- way more than 10 things. - this.setMaxListeners(500); - // this getNatsEnv only ever returns *ONE* connection - this.getNatsEnv = reuseInFlight(async () => { + // many things potentially listen for these events -- way more than 10 things. + this.setMaxListeners(1000); + // this.conat only ever returns *ONE* connection + this.conat = () => { if (this.state == "closed") { throw Error("client already closed"); } - if (this.env) { - return this.env; + if (this.conatClient) { + return this.conatClient; } - this.env = await client.getNatsEnv(); - this.monitorConnectionState(this.env.nc); - return this.env; - }); + this.conatClient = client.conat(); + return this.conatClient; + }; this.account_id = client.account_id; this.project_id = client.project_id; this.compute_server_id = client.compute_server_id; @@ -72,6 +68,10 @@ export class ClientWithState extends EventEmitter { this._reconnect = client.reconnect; } + numSubscriptions = () => { + this.conatClient?.numSubscriptions() ?? 0; + }; + reconnect = async () => { await this._reconnect?.(); }; @@ -85,10 +85,10 @@ export class ClientWithState extends EventEmitter { }; close = () => { - this.env?.nc.close(); + this.conatClient?.close(); this.setConnectionState("closed"); this.removeAllListeners(); - delete this.env; + delete this.conatClient; }; private setConnectionState = (state: State) => { @@ -99,26 +99,10 @@ export class ClientWithState extends EventEmitter { this.emit(state); this.emit("state", state); }; - - private monitorConnectionState = async (nc) => { - this.setConnectionState("connected"); - - for await (const { type } of nc.status()) { - if (this.state == "closed") { - return; - } - if (type.includes("ping") || type == "update" || type == "reconnect") { - // connection is working well - this.setConnectionState("connected"); - } else if (type == "reconnecting") { - this.setConnectionState("connecting"); - } - } - }; } -// do NOT do this until some explicit use of nats is initiated, since we shouldn't -// connect to nats until something tries to do so. +// do NOT do this until some explicit use of conat is initiated, since we shouldn't +// connect to conat until something tries to do so. let timeInitialized = false; function initTime() { if (timeInitialized) { @@ -129,7 +113,7 @@ function initTime() { } let globalClient: null | ClientWithState = null; -export function setNatsClient(client: Client) { +export function setConatClient(client: Client) { globalClient = new ClientWithState(client); } @@ -137,17 +121,17 @@ export async function reconnect() { await globalClient?.reconnect(); } -export const getEnv = reuseInFlight(async () => { +export const conat: () => ConatClient = () => { if (globalClient == null) { - throw Error("must set the global NATS client"); + throw Error("must set the global Conat client"); } initTime(); - return await globalClient.getNatsEnv(); -}); + return globalClient.conat(); +}; export function getClient(): ClientWithState { if (globalClient == null) { - throw Error("must set the global NATS client"); + throw Error("must set the global Conat client"); } initTime(); return globalClient; @@ -176,34 +160,13 @@ export function getLogger(name) { } catch {} // make logger that starts working after global client is set const logger: any = {}; - for (const s of ["debug", "info", "warn"]) { + for (const s of ["debug", "info", "warn", "silly"]) { logger[s] = tmpLogger(s, name, logger); } + logger.silly = logger.debug; return logger; } -// this is a singleton -let theConnection: NatsConnection | null = null; -export const getConnection = reuseInFlight( - async (): Promise => { - if (theConnection == null) { - const { nc } = await getEnv(); - if (nc == null) { - throw Error("bug"); - } - theConnection = nc; - } - return theConnection; - }, -); - -export function getConnectionSync(): NatsConnection | null { - return theConnection; -} - export function numSubscriptions(): number { - if (theConnection == null) { - return 0; - } - return (theConnection as any).protocol.subscriptions.subs.size; + return globalClient?.numSubscriptions() ?? 0; } diff --git a/src/packages/nats/compute/README.md b/src/packages/conat/compute/README.md similarity index 100% rename from src/packages/nats/compute/README.md rename to src/packages/conat/compute/README.md diff --git a/src/packages/nats/compute/manager.ts b/src/packages/conat/compute/manager.ts similarity index 73% rename from src/packages/nats/compute/manager.ts rename to src/packages/conat/compute/manager.ts index 9f33ff85b2..4f21918d80 100644 --- a/src/packages/nats/compute/manager.ts +++ b/src/packages/conat/compute/manager.ts @@ -5,19 +5,18 @@ is used to edit a given file. Access this in the browser for the project you have open: -> m = await cc.client.nats_client.computeServerManager({project_id:cc.current().project_id}) +> m = await cc.client.conat_client.computeServerManager({project_id:cc.current().project_id}) */ -import { dkv, type DKV } from "@cocalc/nats/sync/dkv"; -import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; +import { dkv, type DKV } from "@cocalc/conat/sync/dkv"; import { EventEmitter } from "events"; -import { delay } from "awaiting"; +import { once, until } from "@cocalc/util/async-utils"; type State = "init" | "connected" | "closed"; export interface Info { - // compute server where this path should be opened + // id = compute server where this path should be opened id: number; } @@ -28,9 +27,7 @@ export interface Options { } export function computeServerManager(options: Options) { - const M = new ComputeServerManager(options); - M.init(); - return M; + return new ComputeServerManager(options); } export class ComputeServerManager extends EventEmitter { @@ -48,37 +45,53 @@ export class ComputeServerManager extends EventEmitter { waitUntilReady = async () => { if (this.state == "closed") { - throw Error("manager is closed"); + throw Error("closed"); } else if (this.state == "connected") { return; } - await this.init(); + await once(this, "connected"); }; save = async () => { await this.dkv?.save(); }; - init = reuseInFlight(async () => { - let wait = 3000; - while (this.state == "init") { - try { + private initialized = false; + init = async () => { + if (this.initialized) { + throw Error("init can only be called once"); + } + this.initialized = true; + await until( + async () => { + if (this.state != "init") { + return true; + } const d = await dkv({ name: "compute-server-manager", ...this.options, }); + if (this.state == ("closed" as any)) { + d.close(); + return true; + } this.dkv = d; d.on("change", this.handleChange); this.setState("connected"); - } catch (err) { - wait = Math.min(15000, wait * 1.3) + Math.random(); - console.log( - `WARNING: temporary issue creating compute server manager -- ${err} -- will retry in ${Math.round(wait/1000)}s`, - ); - await delay(wait); - } - } - }); + return true; + }, + { + start: 3000, + decay: 1.3, + max: 15000, + log: (...args) => + console.log( + "WARNING: issue creating compute server manager", + ...args, + ), + }, + ); + }; private handleChange = ({ key: path, value, prev }) => { this.emit("change", { @@ -89,7 +102,7 @@ export class ComputeServerManager extends EventEmitter { }; close = () => { - // console.log("closing a compute server manager"); + // console.log("close compute server manager", this.options); if (this.dkv != null) { this.dkv.removeListener("change", this.handleChange); this.dkv.close(); @@ -113,8 +126,6 @@ export class ComputeServerManager extends EventEmitter { return this.dkv; }; - // Modern sync API: used in backend. - set = (path, id) => { const kv = this.getDkv(); if (!id) { @@ -146,14 +157,22 @@ export class ComputeServerManager extends EventEmitter { path: string; id: number; }) => { - await this.waitUntilReady(); + try { + await this.waitUntilReady(); + } catch { + return; + } this.set(path, id); }; // Call this if you want no compute servers to provide the backend server // for given path. disconnectComputeServer = async ({ path }: { path: string }) => { - await this.waitUntilReady(); + try { + await this.waitUntilReady(); + } catch { + return; + } this.delete(path); }; @@ -161,7 +180,11 @@ export class ComputeServerManager extends EventEmitter { // path, if one is set. Otherwise, return undefined // if nothing is explicitly set for this path (i.e., usually means home base). getServerIdForPath = async (path: string): Promise => { - await this.waitUntilReady(); + try { + await this.waitUntilReady(); + } catch { + return; + } return this.get(path); }; @@ -171,6 +194,9 @@ export class ComputeServerManager extends EventEmitter { path: string, ): Promise<{ [path: string]: number }> => { await this.waitUntilReady(); + if (this.state == "closed") { + throw Error("closed"); + } const kv = this.getDkv(); const v: { [path: string]: number } = {}; const slash = path.endsWith("/") ? path : path + "/"; diff --git a/src/packages/conat/core/client.ts b/src/packages/conat/core/client.ts new file mode 100644 index 0000000000..a6025435b2 --- /dev/null +++ b/src/packages/conat/core/client.ts @@ -0,0 +1,1768 @@ +/* +core/client.s -- core conat client + +This is a client that has a similar API to NATS / Socket.io, but is much, +much better in so many ways: + +- It has global pub/sub just like with NATS. This uses the server to + rebroadcast messages, and for authentication. + Better than NATS: Authentication is done for a subject *as + needed* instead of at connection time. + +- Message can be arbitrarily large and they are *automatically* divided + into chunks and reassembled. Better than both NATS and socket.io. + +- There are multiple supported ways of encoding messages, and + no coordination is required with the server or other clients! E.g., + one message can be sent with one encoding and the next with a different + encoding and that's fine. + - MsgPack: https://msgpack.org/ -- a very compact encoding that handles + dates nicely and small numbers efficiently. This also works + well with binary Buffer objects, which is nice. + - JsonCodec: uses JSON.stringify and TextEncoder. This does not work + with Buffer or Date and is less compact, but can be very fast. + + +THE CORE API + +This section contains the crucial information you have to know to build a distributed +system using Conat. It's our take on the NATS primitives (it's not exactly the +same, but it is close). It's basically a symmetrical pub/sub/reqest/respond model +for messaging on which you can build distributed systems. The tricky part, which +NATS.js gets wrong (in my opinion), is implementing this in a way that is robust +and scalable, in terms for authentication, real world browser connectivity and +so on. Our approach is to use proven mature technology like socket.io, sqlite +and valkey, instead of writing everything from scratch. + +Clients: We view all clients as plugged into a common "dial tone", +except for optional permissions that are configured when starting the server. +The methods you call on the client to build everything are: + + - subscribe, subscribeSync - subscribe to a subject which returns an + async iterator over all messages that match the subject published by + anyone with permission to do so. If you provide the same optional + queue parameter for multiple subscribers, then one subscriber in each queue + group receives each message. The async form of this functino confirms + the subscription was created before returning. If a client creates multiple + subscriptions at the same time, the queue group must be the same. + Subscriptions are guaranteed to stay valid until the client ends them; + they do not stop working due to client or server reconnects or restarts. + (If you need more subscriptions with different queue groups, make another + client object.) + + - publish, publishSync - publish to a subject. The async version returns + a count of the number of recipients, whereas the sync version is + fire-and-forget. + **There is no a priori size limit on messages since chunking + is automatic. However, we have to impose some limit, but + it can be much larger than the socketio message size limit.** + + - request - send a message to a subject, and if there is at least one + subscriber listening, it may respond. If there are no subscribers, + it throws a 503 error. To create a microservice, subscribe + to a subject pattern and called mesg.respond(...) on each message you + receive. + + - requestMany - send a message to a subject, and receive many + messages in reply. Typically you end the response stream by sending + a null message, but what you do is up to you. This is very useful + for streaming arbitrarily large data, long running changefeeds, LLM + responses, etc. + + +Messages: A message mesg is: + + - Data: + - subject - the subject the message was sent to + - encoding - usually MessagePack + - raw - encoded binary data + - headers - a JSON-able Javascript object. + + - Methods: + - data: this is a property, so if you do mesg.data, then it decodes raw + and returns the resulting Javascript object. + - respond, respondSync: if REPLY_HEADER is set, calling this publishes a + respond message to the original sender of the message. + + +Persistence: + +We also implement persistent streams, where you can also set a key. This can +be used to build the analogue of Jetstream's streams and kv stores. The object +store isn't necessary since there is no limit on message size. Conat's persistent +streams are compressed by default and backed by individual sqlite files, which +makes them very memory efficient and it is easy to tier storage to cloud storage. + +UNIT TESTS: See packages/server/conat/test/core + +MISC NOTES: + +NOTE: There is a socketio msgpack parser, but it just doesn't +work at all, which is weird. Also, I think it's impossible to +do the sort of chunking we want at the level of a socket.io +parser -- it's just not possible in that the encoding. We customize +things purely client side without using a parser, and get a much +simpler and better result, inspired by how NATS approaches things +with opaque messages. + + +SUBSCRIPTION ROBUSTNESS: When you call client.subscribe(...) you get back an async iterator. +It ONLY ends when you explicitly do the standard ways of terminating +such an iterator, including calling .close() on it. It is a MAJOR BUG +if it were to terminate for any other reason. In particular, the subscription +MUST NEVER throw an error or silently end when the connection is dropped +then resumed, or the server is restarted, or the client connects to +a different server! These situations can, of course, result in missing +some messages, but that's understood. There are no guarantees at all with +a subscription that every message is received. That said, we have enabled +connectionStateRecovery (and added special conat support for it) so no messages +are dropped for temporary disconnects, even up to several minutes, +and even in valkey cluster mode! Finally, any time a client disconnects +and reconnects, the client ensures that all subscriptions exist for it on the server +via a sync process. + +Subscription robustness is a major difference with NATS.js, which would +mysteriously terminate subscriptions for a variety of reasons, meaning that any +code using subscriptions had to be wrapped in ugly complexity to be +usable in production. + +USAGE: + +The following should mostly work to interactively play around with this +code and develop it. It's NOT automatically tested and depends on your +environment though, so may break. See the unit tests in + + packages/server/conat/test/core/ + +for something that definitely works perfectly. + + +For developing at the command line, cd to packages/backend, then in node: + + c = require('@cocalc/backend/conat/conat').connect() + +or + + c = require('@cocalc/conat/core/client').connect('http://localhost:3000') + + c.watch('a') + + s = await c.subscribe('a'); for await (const x of s) { console.log(x.length)} + +// in another console + + c = require('@cocalc/backend/conat/conat').connect() + c.publish('a', 'hello there') + +// in browser (right now) + + cc.conat.conat() + +// client server: + + s = await c.subscribe('eval'); for await(const x of s) { x.respond(eval(x.data)) } + +then in another console + + f = async () => (await c.request('eval', '2+3')).data + await f() + + t = Date.now(); for(i=0;i<1000;i++) { await f()} ; Date.now()-t + +// slower, but won't silently fail due to errors, etc. + + f2 = async () => (await c.request('eval', '2+3', {confirm:true})).data + +Wildcard subject: + + + c = require('@cocalc/conat/core/client').connect(); c.watch('a.*'); + + + c = require('@cocalc/conat/core/client').connect(); c.publish('a.x', 'foo') + + +Testing disconnect + + c.sub('>') + c.conn.io.engine.close();0; + +other: + + a=0; setInterval(()=>c.pub('a',a++), 250) + +*/ + +import { + connect as connectToSocketIO, + type SocketOptions, + type ManagerOptions, +} from "socket.io-client"; +import { EventIterator } from "@cocalc/util/event-iterator"; +import type { ConnectionStats, ServerInfo } from "./types"; +import * as msgpack from "@msgpack/msgpack"; +import { randomId } from "@cocalc/conat/names"; +import type { JSONValue } from "@cocalc/util/types"; +import { EventEmitter } from "events"; +import { callback } from "awaiting"; +import { + isValidSubject, + isValidSubjectWithoutWildcards, +} from "@cocalc/conat/util"; +import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; +import { once, until } from "@cocalc/util/async-utils"; +import { delay } from "awaiting"; +import { getLogger } from "@cocalc/conat/client"; +import { refCacheSync } from "@cocalc/util/refcache"; +import { join } from "path"; +import { dko, type DKO } from "@cocalc/conat/sync/dko"; +import { dkv, type DKVOptions, type DKV } from "@cocalc/conat/sync/dkv"; +import { + dstream, + type DStreamOptions, + type DStream, +} from "@cocalc/conat/sync/dstream"; +import { akv, type AKV } from "@cocalc/conat/sync/akv"; +import { astream, type AStream } from "@cocalc/conat/sync/astream"; +import TTL from "@isaacs/ttlcache"; +import { + ConatSocketServer, + ConatSocketClient, + ServerSocket, + type SocketConfiguration, +} from "@cocalc/conat/socket"; +export { type ConatSocketServer, ConatSocketClient, ServerSocket }; +import { + type SyncTableOptions, + type ConatSyncTable, + createSyncTable, +} from "@cocalc/conat/sync/synctable"; + +export const MAX_INTEREST_TIMEOUT = 90000; + +const MSGPACK_ENCODER_OPTIONS = { + // ignoreUndefined is critical so database queries work properly, and + // also we have a lot of api calls with tons of wasted undefined values. + ignoreUndefined: true, +}; + +export const STICKY_QUEUE_GROUP = "sticky"; + +export const DEFAULT_SOCKETIO_CLIENT_OPTIONS = { + // A major problem if we allow long polling is that we must always use at most + // half the chunk size... because there is no way to know if recipients will be + // using long polling to RECEIVE messages. Not insurmountable. + transports: ["websocket"], + + // nodejs specific for project/compute server in some settings + rejectUnauthorized: false, + + reconnection: true, + reconnectionDelay: 1000, + reconnectionDelayMax: 15000, + reconnectionAttempts: 9999999999, // infinite +}; + +type State = "disconnected" | "connected" | "closed"; + +const logger = getLogger("core/client"); + +interface Options { + // address = the address of a cocalc server, including the base url, e.g., + // + // https://cocalc.com + // + // or for a dev server running locally with a base url: + // + // http://localhost:4043/3fa218e5-7196-4020-8b30-e2127847cc4f/port/5002 + // + // The socketio path is always /conat (after the base url) and is set automatically. + // + address?: string; + inboxPrefix?: string; +} + +export type ClientOptions = Options & { + noCache?: boolean; +} & Partial & + Partial; + +const INBOX_PREFIX = "_INBOX"; +const REPLY_HEADER = "CN-Reply"; +const MAX_HEADER_SIZE = 100000; + +const STATS_LOOP = 5000; + +export let DEFAULT_REQUEST_TIMEOUT = 7500; +export let DEFAULT_PUBLISH_TIMEOUT = 7500; + +export function setDefaultTimeouts({ + request = DEFAULT_REQUEST_TIMEOUT, + publish = DEFAULT_PUBLISH_TIMEOUT, +}: { + request?: number; + publish?: number; +}) { + DEFAULT_REQUEST_TIMEOUT = request; + DEFAULT_PUBLISH_TIMEOUT = publish; +} + +export enum DataEncoding { + MsgPack = 0, + JsonCodec = 1, +} + +interface SubscriptionOptions { + maxWait?: number; + mesgLimit?: number; + queue?: string; + // sticky: when a choice from a queue group is made, the same choice is always made + // in the future for any message with subject matching subject with the last segment + // replaced by a *, until the target goes away. Setting this just sets the queue + // option to STICKY_QUEUE_GROUP. + // + // Examples of subjects matching except possibly last segment are + // foo.bar.lJcBSieLn and foo.bar.ZzsDC376ge + // You can put anything random in the last segment and all messages + // that match foo.bar.* get the same choice from the queue group. + // The idea is that *when* the message with subject foo.bar.lJcBSieLn gets + // sent, the backend server selects a target from the queue group to receive + // that message. It remembers the choice, and so long as that target is subscribed, + // it sends any message matching foo.bar.* to that same target. + // This is used in our implementation of persistent socket connections that + // are built on pub/sub. The underlying implementation uses consistent + // hashing and messages to sync state of the servers. + sticky?: boolean; + respond?: Function; + timeout?: number; +} + +// WARNING! This is the default and you can't just change it! +// Yes, for specific messages you can, but in general DO NOT. The reason is because, e.g., +// JSON will turn Dates into strings, and we no longer fix that. So unless you modify the +// JsonCodec to handle Date's properly, don't change this!! +const DEFAULT_ENCODING = DataEncoding.MsgPack; + +function cocalcServerToSocketioAddress(url?: string): { + address: string; + path: string; +} { + url = url ?? process.env.CONAT_SERVER; + if (!url) { + throw Error( + "Must give Conat server address or set CONAT_SERVER environment variable", + ); + } + const u = new URL(url, "http://dummy.org"); + const address = u.origin; + const path = join(u.pathname, "conat"); + return { address, path }; +} + +const cache = refCacheSync({ + name: "conat-client", + createObject: (opts: ClientOptions) => { + return new Client(opts); + }, +}); + +export function connect(opts: ClientOptions = {}) { + if (!opts.address) { + const x = cache.one(); + if (x != null) { + return x; + } + } + return cache(opts); +} + +// Get any cached client, if there is one; otherwise make one +// with default options. +export function getClient() { + return cache.one() ?? connect(); +} + +export class Client extends EventEmitter { + public conn: ReturnType; + // queueGroups is a map from subject to the queue group for the subscription to that subject + private queueGroups: { [subject: string]: string } = {}; + private subs: { [subject: string]: SubscriptionEmitter } = {}; + private sockets: { + // all socket servers created using this Client + servers: { [subject: string]: ConatSocketServer }; + // all client connections created using this Client. + clients: { [subject: string]: { [id: string]: ConatSocketClient } }; + } = { servers: {}, clients: {} }; + private readonly options: ClientOptions; + private inboxSubject: string; + private inbox?: EventEmitter; + private permissionError = { + pub: new TTL({ ttl: 1000 * 60 }), + sub: new TTL({ ttl: 1000 * 60 }), + }; + public info: ServerInfo | undefined = undefined; + // total number of + public readonly stats: ConnectionStats & { + recv0: { messages: number; bytes: number }; + } = { + send: { messages: 0, bytes: 0 }, + recv: { messages: 0, bytes: 0 }, + // recv0 = count since last connect + recv0: { messages: 0, bytes: 0 }, + subs: 0, + }; + + public readonly id: string = randomId(); + public state: State = "disconnected"; + + constructor(options: ClientOptions) { + super(); + this.setMaxListeners(1000); + this.options = options; + + // for socket.io the address has no base url + const { address, path } = cocalcServerToSocketioAddress( + this.options.address, + ); + logger.debug(`Conat: Connecting to ${this.options.address}...`); + // if (options.extraHeaders == null) { + // console.trace("WARNING: no auth set"); + // } + this.conn = connectToSocketIO(address, { + ...DEFAULT_SOCKETIO_CLIENT_OPTIONS, + ...options, + path, + }); + + this.conn.on("info", (info) => { + const firstTime = this.info == null; + this.info = info; + this.emit("info", info); + setTimeout(this.syncSubscriptions, firstTime ? 3000 : 0); + }); + this.conn.on("permission", ({ message, type, subject }) => { + logger.debug(message); + this.permissionError[type]?.set(subject, message); + }); + this.conn.on("connect", async () => { + logger.debug(`Conat: Connected to ${this.options.address}`); + if (this.conn.connected) { + this.setState("connected"); + } + }); + this.conn.io.on("error", (...args) => { + logger.debug( + `Conat: Error connecting to ${this.options.address} -- `, + ...args, + ); + }); + this.conn.on("disconnect", () => { + this.stats.recv0 = { messages: 0, bytes: 0 }; // reset on disconnect + this.setState("disconnected"); + this.disconnectAllSockets(); + }); + this.initInbox(); + this.statsLoop(); + } + + disconnect = () => { + this.disconnectAllSockets(); + // @ts-ignore + setTimeout(() => this.conn.io.disconnect(), 1); + }; + + waitUntilSignedIn = reuseInFlight(async () => { + // not "signed in" if -- + // - not connected, or + // - no info at all (which gets sent on sign in) + // - or the user is {error:....}, which is what happens when sign in fails + // e.g., do to an expired cookie + if ( + this.info == null || + this.state != "connected" || + this.info?.user?.error + ) { + await once(this, "info"); + } + }); + + private statsLoop = async () => { + await until( + async () => { + if (this.isClosed()) { + return true; + } + try { + await this.waitUntilConnected(); + if (this.isClosed()) { + return true; + } + this.conn.emit("stats", { recv0: this.stats.recv0 }); + } catch {} + return false; + }, + { start: STATS_LOOP, max: STATS_LOOP }, + ); + }; + + interest = async (subject: string): Promise => { + return await this.waitForInterest(subject, { timeout: 0 }); + }; + + waitForInterest = async ( + subject: string, + { + timeout = MAX_INTEREST_TIMEOUT, + }: { + timeout?: number; + } = {}, + ) => { + if (!isValidSubjectWithoutWildcards(subject)) { + throw Error( + `subject ${subject} must be a valid subject without wildcards`, + ); + } + if (timeout > MAX_INTEREST_TIMEOUT) { + throw Error(`timeout must be at most ${MAX_INTEREST_TIMEOUT}`); + } + const f = (cb) => { + this.conn + .timeout(timeout ? timeout : 10000) + .emit("wait-for-interest", { subject, timeout }, (err, response) => { + if (err) { + cb(err); + } else if (response.error) { + cb(new ConatError(response.error, { code: response.code })); + } else { + cb(undefined, response); + } + }); + }; + return await callback(f); + }; + + recvStats = (bytes: number) => { + this.stats.recv.messages += 1; + this.stats.recv.bytes += bytes; + this.stats.recv0.messages += 1; + this.stats.recv0.bytes += bytes; + }; + + // There should usually be no reason to call this because socket.io + // is so good at abstracting this away. It's useful for unit testing. + waitUntilConnected = reuseInFlight(async () => { + if (this.conn.connected) { + return; + } + // @ts-ignore + await once(this.conn, "connect"); + }); + + waitUntilReady = reuseInFlight(async () => { + await this.waitUntilSignedIn(); + await this.waitUntilConnected(); + }); + + private setState = (state: State) => { + if (this.isClosed() || this.state == state) { + return; + } + this.state = state; + this.emit(state); + }; + + private temporaryInboxSubject = () => { + if (!this.inboxSubject) { + throw Error("inbox not setup properly"); + } + return `${this.inboxSubject}.${randomId()}`; + }; + + private getInbox = reuseInFlight(async (): Promise => { + if (this.inbox == null) { + if (this.isClosed()) { + throw Error("closed"); + } + await once(this, "inbox"); + } + if (this.inbox == null) { + throw Error("bug"); + } + return this.inbox; + }); + + private initInbox = async () => { + // For request/respond instead of setting up one + // inbox *every time there is a request*, we setup a single + // inbox once and for all for all responses. We listen for + // everything to inbox...Prefix.* and emit it via this.inbox. + // The request sender then listens on this.inbox for the response. + // We *could* use a regular subscription for each request, + // but (1) that massively increases the load on the server for + // every single request (having to create and destroy subscriptions) + // and (2) there is a race condition between creating that subscription + // and getting the response; it's fine with one server, but with + // multiple servers solving the race condition would slow everything down + // due to having to wait for so many acknowledgements. Instead, we + // remove all those problems by just using a single inbox subscription. + const inboxPrefix = this.options.inboxPrefix ?? INBOX_PREFIX; + if (!inboxPrefix.startsWith(INBOX_PREFIX)) { + throw Error(`custom inboxPrefix must start with '${INBOX_PREFIX}'`); + } + this.inboxSubject = `${inboxPrefix}.${randomId()}`; + let sub; + await until( + async () => { + try { + await this.waitUntilSignedIn(); + sub = await this.subscribe(this.inboxSubject + ".*"); + return true; + } catch (err) { + if (this.isClosed()) { + return true; + } + // this should only fail due to permissions issues, at which point + // request can't work, but pub/sub can. + console.log(`WARNING: inbox not available -- ${err}`); + } + return false; + }, + { start: 1000, max: 15000 }, + ); + if (this.isClosed()) { + return; + } + + this.inbox = new EventEmitter(); + (async () => { + for await (const mesg of sub) { + if (this.inbox == null) { + return; + } + this.inbox.emit(mesg.subject, mesg); + } + })(); + this.emit("inbox", this.inboxSubject); + }; + + private isClosed = () => { + return this.state == "closed"; + }; + + close = () => { + if (this.isClosed()) { + return; + } + this.setState("closed"); + this.removeAllListeners(); + this.closeAllSockets(); + // @ts-ignore + delete this.sockets; + for (const subject in this.queueGroups) { + this.conn.emit("unsubscribe", { subject }); + delete this.queueGroups[subject]; + } + for (const sub of Object.values(this.subs)) { + sub.refCount = 0; + sub.close(); + // @ts-ignore + delete this.subs; + } + // @ts-ignore + delete this.queueGroups; + // @ts-ignore + delete this.inboxSubject; + delete this.inbox; + // @ts-ignore + delete this.options; + // @ts-ignore + delete this.info; + // @ts-ignore + delete this.permissionError; + + try { + this.conn.close(); + } catch {} + }; + + private syncSubscriptions = reuseInFlight(async () => { + let fails = 0; + await until( + async () => { + if (this.isClosed()) return true; + try { + if (this.info == null) { + // no point in trying until we are signed in and connected + await once(this, "info"); + } + if (this.isClosed()) return true; + await this.waitUntilConnected(); + if (this.isClosed()) return true; + const stable = await this.syncSubscriptions0(10000); + if (stable) { + return true; + } + } catch (err) { + fails++; + if (fails >= 3) { + console.log( + `WARNING: failed to sync subscriptions ${fails} times -- ${err}`, + ); + } + } + return false; + }, + { start: 1000, max: 15000 }, + ); + }); + + // syncSubscriptions0 ensures that we're subscribed on server + // to what we think we're subscribed to, or throws an error. + private syncSubscriptions0 = async (timeout: number): Promise => { + if (this.isClosed()) return true; + if (this.info == null) { + throw Error("not signed in"); + } + const subs = await this.getSubscriptions(timeout); + // console.log("syncSubscriptions", { + // server: subs, + // client: Object.keys(this.queueGroups), + // }); + const missing: { subject: string; queue: string }[] = []; + for (const subject in this.queueGroups) { + // subscribe on backend to all subscriptions we think we should have that + // the server does not have + if (!subs.has(subject)) { + missing.push({ + subject, + queue: this.queueGroups[subject], + }); + } + } + let stable = true; + if (missing.length > 0) { + stable = false; + const resp = await callback( + this.conn.timeout(timeout).emit.bind(this.conn), + "subscribe", + missing, + ); + // some subscription could fail due to permissions changes, e.g., user got + // removed from a project. + for (let i = 0; i < missing.length; i++) { + if (resp[i].error) { + const sub = this.subs[missing[i].subject]; + if (sub != null) { + sub.close(true); + } + } + } + } + const extra: { subject: string }[] = []; + for (const subject in subs) { + if (this.queueGroups[subject] != null) { + // server thinks we're subscribed but we do not think so, so cancel that + extra.push({ subject }); + } + } + if (extra.length > 0) { + await callback( + this.conn.timeout(timeout).emit.bind(this.conn), + "unsubscribe", + extra, + ); + stable = false; + } + return stable; + }; + + numSubscriptions = () => Object.keys(this.queueGroups).length; + + private getSubscriptions = async ( + timeout = DEFAULT_REQUEST_TIMEOUT, + ): Promise> => { + const subs = await callback( + this.conn.timeout(timeout).emit.bind(this.conn), + "subscriptions", + null, + ); + return new Set(subs); + }; + + // returns EventEmitter that emits 'message', mesg: Message + private subscriptionEmitter = ( + subject: string, + { + closeWhenOffCalled, + queue, + sticky, + confirm, + timeout, + }: { + // if true, when the off method of the event emitter is called, then + // the entire subscription is closed. This is very useful when we wrap the + // EvenEmitter in an async iterator. + closeWhenOffCalled?: boolean; + + // the queue group -- if not given, then one is randomly assigned. + queue?: string; + + // if true, sets queue to "sticky" + sticky?: boolean; + + // confirm -- get confirmation back from server that subscription was created + confirm?: boolean; + + // how long to wait to confirm creation of the subscription; + // only used when confirm=true. + timeout?: number; + } = {}, + ): { sub: SubscriptionEmitter; promise? } => { + if (this.isClosed()) { + throw Error("closed"); + } + if (!isValidSubject(subject)) { + throw Error(`invalid subscribe subject '${subject}'`); + } + if (this.permissionError.sub.has(subject)) { + const message = this.permissionError.sub.get(subject)!; + logger.debug(message); + throw new ConatError(message, { code: 403 }); + } + if (sticky) { + if (queue) { + throw Error("must not specify queue group if sticky is true"); + } + queue = STICKY_QUEUE_GROUP; + } + let sub = this.subs[subject]; + if (sub != null) { + if (queue && this.queueGroups[subject] != queue) { + throw Error( + `client can only have one queue group subscription for a given subject -- subject='${subject}', queue='${queue}'`, + ); + } + if (queue == STICKY_QUEUE_GROUP) { + throw Error( + `can only have one sticky subscription per client -- subject='${subject}'`, + ); + } + sub.refCount += 1; + return { sub, promise: undefined }; + } + if (this.queueGroups[subject] != null) { + throw Error(`already subscribed to '${subject}'`); + } + if (!queue) { + queue = randomId(); + } + this.queueGroups[subject] = queue; + sub = new SubscriptionEmitter({ + client: this, + subject, + closeWhenOffCalled, + }); + this.subs[subject] = sub; + this.stats.subs++; + let promise; + if (confirm) { + const f = (cb) => { + const handle = (response) => { + if (response?.error) { + cb(new ConatError(response.error, { code: response.code })); + } else { + cb(response?.error, response); + } + }; + if (timeout) { + this.conn + .timeout(timeout) + .emit("subscribe", { subject, queue }, (err, response) => { + if (err) { + handle({ error: `${err}`, code: 408 }); + } else { + handle(response); + } + }); + } else { + this.conn.emit("subscribe", { subject, queue }, handle); + } + }; + promise = callback(f); + } else { + this.conn.emit("subscribe", { subject, queue }); + promise = undefined; + } + sub.once("closed", () => { + if (this.isClosed()) { + return; + } + this.conn.emit("unsubscribe", { subject }); + delete this.queueGroups[subject]; + if (this.subs[subject] != null) { + this.stats.subs--; + delete this.subs[subject]; + } + }); + return { sub, promise }; + }; + + private subscriptionIterator = ( + sub, + opts?: SubscriptionOptions, + ): Subscription => { + // @ts-ignore + const iter = new EventIterator(sub, "message", { + idle: opts?.maxWait, + limit: opts?.mesgLimit, + map: (args) => args[0], + }); + return iter; + }; + + subscribeSync = ( + subject: string, + opts?: SubscriptionOptions, + ): Subscription => { + const { sub } = this.subscriptionEmitter(subject, { + confirm: false, + closeWhenOffCalled: true, + sticky: opts?.sticky, + queue: opts?.queue, + }); + return this.subscriptionIterator(sub, opts); + }; + + subscribe = async ( + subject: string, + opts?: SubscriptionOptions, + ): Promise => { + await this.waitUntilSignedIn(); + const { sub, promise } = this.subscriptionEmitter(subject, { + confirm: true, + closeWhenOffCalled: true, + queue: opts?.queue, + sticky: opts?.sticky, + timeout: opts?.timeout, + }); + await promise; + return this.subscriptionIterator(sub, opts); + }; + + sub = this.subscribe; + + /* + A service is a subscription with a function to respond to requests by name. + Call service with an implementation: + + service = await client1.service('arith', {mul : async (a,b)=>{a*b}, add : async (a,b)=>a+b}) + + Use the service: + + arith = await client2.call('arith') + await arith.mul(2,3) + await arith.add(2,3) + + There's by default a single queue group '0', so if you create multiple services on various + computers, then requests are load balanced across them automatically. Explicitly set + a random queue group (or something else) and use callMany if you don't want this behavior. + + Close the service when done: + + service.close(); + + See backend/conat/test/core/services.test.ts for a tested and working example + that involves typescript and shows how to use wildcard subjects and get the + specific subject used for a call by using that this is bound to the calling mesg. + */ + service: ( + subject: string, + impl: T, + opts?: SubscriptionOptions, + ) => Promise = async (subject, impl, opts) => { + const sub = await this.subscribe(subject, { + ...opts, + queue: opts?.queue ?? "0", + }); + const respond = async (mesg: Message) => { + try { + const [name, args] = mesg.data; + // call impl[name], but with 'this' set to the object {subject:...}, + // so inside the service, it is possible to know what subject was used + // in the request, in case subject is a wildcard subject. + // const result = await impl[name].apply( + // { subject: mesg.subject }, + // ...args, + // ); + // const result = await impl[name].apply( + // { subject: mesg.subject }, + // ...args, + // ); + // mesg.respondSync(result); + let f = impl[name]; + if (f == null) { + throw Error(`${name} not defined`); + } + const result = await f.apply(mesg, args); + mesg.respondSync(result); + } catch (err) { + mesg.respondSync(null, { headers: { error: `${err}` } }); + } + }; + const loop = async () => { + // todo -- param to set max number of responses at once. + for await (const mesg of sub) { + respond(mesg); + } + }; + loop(); + return sub; + }; + + // Call a service as defined above. + call(subject: string, opts?: PublishOptions): T { + const call = async (name: string, args: any[]) => { + const resp = await this.request(subject, [name, args], opts); + if (resp.headers?.error) { + throw Error(`${resp.headers.error}`); + } else { + return resp.data; + } + }; + + return new Proxy( + {}, + { + get: (_, name) => { + if (typeof name !== "string") { + return undefined; + } + return async (...args) => await call(name, args); + }, + }, + ) as T; + } + + callMany(subject: string, opts?: RequestManyOptions): T { + const maxWait = opts?.maxWait ? opts?.maxWait : DEFAULT_REQUEST_TIMEOUT; + const self = this; + async function* callMany(name: string, args: any[]) { + const sub = await self.requestMany(subject, [name, args], { + ...opts, + maxWait, + }); + for await (const resp of sub) { + if (resp.headers?.error) { + yield new ConatError(`${resp.headers.error}`, { + code: resp.headers.code, + }); + } else { + yield resp.data; + } + } + } + + return new Proxy( + {}, + { + get: (_, name) => { + if (typeof name !== "string") { + return undefined; + } + return async (...args) => await callMany(name, args); + }, + }, + ) as T; + } + + publishSync = ( + subject: string, + mesg, + opts?: PublishOptions, + ): { bytes: number } => { + if (this.isClosed()) { + // already closed + return { bytes: 0 }; + } + return this._publish(subject, mesg, opts); + }; + + publish = async ( + subject: string, + mesg, + opts?: PublishOptions, + ): Promise<{ + // bytes encoded (doesn't count some extra wrapping) + bytes: number; + // count is the number of matching subscriptions + // that the server *sent* this message to since the server knows about them. + // However, there's no guaranteee that the subscribers actually exist + // **right now** or received these messages. + count: number; + }> => { + if (this.isClosed()) { + // already closed + return { bytes: 0, count: 0 }; + } + await this.waitUntilSignedIn(); + const { bytes, getCount, promise } = this._publish(subject, mesg, { + ...opts, + confirm: true, + }); + await promise; + return { bytes, count: getCount?.()! }; + }; + + private _publish = ( + subject: string, + mesg, + { + headers, + raw, + encoding = DEFAULT_ENCODING, + confirm, + timeout = DEFAULT_PUBLISH_TIMEOUT, + }: PublishOptions & { confirm?: boolean } = {}, + ) => { + if (this.isClosed()) { + return { bytes: 0 }; + } + if (!isValidSubjectWithoutWildcards(subject)) { + throw Error(`invalid publish subject ${subject}`); + } + if (this.permissionError.pub.has(subject)) { + const message = this.permissionError.pub.get(subject)!; + logger.debug(message); + throw new ConatError(message, { code: 403 }); + } + raw = raw ?? encode({ encoding, mesg }); + this.stats.send.messages += 1; + this.stats.send.bytes += raw.length; + + // default to 1MB is safe since it's at least that big. + const chunkSize = Math.max( + 1000, + (this.info?.max_payload ?? 1e6) - MAX_HEADER_SIZE, + ); + let seq = 0; + let id = randomId(); + const promises: any[] = []; + let count = 0; + for (let i = 0; i < raw.length; i += chunkSize) { + // !!FOR TESTING ONLY!! + // if (Math.random() <= 0.01) { + // console.log("simulating a chunk drop", { subject, seq }); + // seq += 1; + // continue; + // } + const done = i + chunkSize >= raw.length ? 1 : 0; + const v: any[] = [ + subject, + id, + seq, + done, + encoding, + raw.slice(i, i + chunkSize), + ]; + if (done && headers) { + v.push(headers); + } + if (confirm) { + let done = false; + const f = (cb) => { + const handle = (response) => { + // console.log("_publish", { done, subject, mesg, headers, confirm }); + if (response?.error) { + cb(new ConatError(response.error, { code: response.code })); + } else { + cb(response?.error, response); + } + }; + if (timeout) { + const timer = setTimeout(() => { + done = true; + cb(new ConatError("timeout", { code: 408 })); + }, timeout); + + this.conn.timeout(timeout).emit("publish", v, (err, response) => { + if (done) { + return; + } + clearTimeout(timer); + if (err) { + handle({ error: `${err}`, code: 408 }); + } else { + handle(response); + } + }); + } else { + this.conn.emit("publish", v, handle); + } + }; + const promise = (async () => { + const response = await callback(f); + count = Math.max(count, response.count ?? 0); + })(); + promises.push(promise); + } else { + this.conn.emit("publish", v); + } + seq += 1; + } + if (confirm) { + return { + bytes: raw.length, + getCount: () => count, + promise: Promise.all(promises), + }; + } + return { bytes: raw.length }; + }; + + pub = this.publish; + + request = async ( + subject: string, + mesg: any, + { + timeout = DEFAULT_REQUEST_TIMEOUT, + // waitForInterest -- if publish fails due to no receivers and + // waitForInterest is true, will wait until there is a receiver + // and publish again: + waitForInterest = false, + ...options + }: PublishOptions & { timeout?: number; waitForInterest?: boolean } = {}, + ): Promise => { + if (timeout <= 0) { + throw Error("timeout must be positive"); + } + const start = Date.now(); + const inbox = await this.getInbox(); + const inboxSubject = this.temporaryInboxSubject(); + const sub = new EventIterator(inbox, inboxSubject, { + idle: timeout, + limit: 1, + map: (args) => args[0], + }); + + const opts = { + ...options, + timeout, + headers: { ...options?.headers, [REPLY_HEADER]: inboxSubject }, + }; + const { count } = await this.publish(subject, mesg, opts); + + if (!count) { + const giveUp = () => { + sub.stop(); + throw new ConatError( + `request -- no subscribers matching '${subject}'`, + { + code: 503, + }, + ); + }; + if (waitForInterest) { + await this.waitForInterest(subject, { timeout }); + if (this.state == "closed") { + throw Error("closed"); + } + const remaining = timeout - (Date.now() - start); + if (remaining <= 1000) { + throw new ConatError("timeout", { code: 408 }); + } + // no error so there is very likely now interest, so we publish again: + const { count } = await this.publish(subject, mesg, { + ...opts, + timeout: remaining, + }); + if (!count) { + giveUp(); + } + } else { + giveUp(); + } + } + + for await (const resp of sub) { + sub.stop(); + return resp; + } + sub.stop(); + throw new ConatError("timeout", { code: 408 }); + }; + + // NOTE: Using requestMany returns a Subscription sub, and + // you can call sub.close(). However, the sender doesn't + // know that this happened and the messages are still going + // to your inbox. Similarly if you set a maxWait, the + // subscription just ends at that point, but the server + // sending messages doesn't know. This is a shortcoming the + // pub/sub model. You must decide entirely based on your + // own application protocol how to terminate. + requestMany = async ( + subject: string, + mesg: any, + { maxMessages, maxWait, ...options }: RequestManyOptions = {}, + ): Promise => { + if (maxMessages != null && maxMessages <= 0) { + throw Error("maxMessages must be positive"); + } + if (maxWait != null && maxWait <= 0) { + throw Error("maxWait must be positive"); + } + const inbox = await this.getInbox(); + const inboxSubject = this.temporaryInboxSubject(); + const sub = new EventIterator(inbox, inboxSubject, { + idle: maxWait, + limit: maxMessages, + map: (args) => args[0], + }); + const { count } = await this.publish(subject, mesg, { + ...options, + headers: { ...options?.headers, [REPLY_HEADER]: inboxSubject }, + }); + if (!count) { + sub.stop(); + throw new ConatError( + `requestMany -- no subscribers matching ${subject}`, + { code: 503 }, + ); + } + return sub; + }; + + // watch: this is mainly for debugging and interactive use. + watch = ( + subject: string, + cb = (x) => console.log(`${new Date()}: ${x.subject}:`, x.data, x.headers), + opts?: SubscriptionOptions, + ) => { + const sub = this.subscribeSync(subject, opts); + const f = async () => { + for await (const x of sub) { + cb(x); + } + }; + f(); + return sub; + }; + + sync = { + dkv: async (opts: DKVOptions): Promise => + await dkv({ ...opts, client: this }), + akv: async (opts: DKVOptions): Promise => + await akv({ ...opts, client: this }), + dko: async (opts: DKVOptions): Promise => + await dko({ ...opts, client: this }), + dstream: async (opts: DStreamOptions): Promise => + await dstream({ ...opts, client: this }), + astream: async (opts: DStreamOptions): Promise => + await astream({ ...opts, client: this }), + synctable: async (opts: SyncTableOptions): Promise => + await createSyncTable({ ...opts, client: this }), + }; + + socket = { + listen: ( + subject: string, + opts?: SocketConfiguration, + ): ConatSocketServer => { + if (this.state == "closed") { + throw Error("closed"); + } + if (this.sockets.servers[subject] !== undefined) { + throw Error( + `there can be at most one socket server per client listening on a subject (subject='${subject}')`, + ); + } + const server = new ConatSocketServer({ + subject, + role: "server", + client: this, + id: this.id, + ...opts, + }); + this.sockets.servers[subject] = server; + server.once("closed", () => { + delete this.sockets.servers[subject]; + }); + return server; + }, + + connect: ( + subject: string, + opts?: SocketConfiguration, + ): ConatSocketClient => { + if (this.state == "closed") { + throw Error("closed"); + } + const id = randomId(); + const client = new ConatSocketClient({ + subject, + role: "client", + client: this, + id, + ...opts, + }); + if (this.sockets.clients[subject] === undefined) { + this.sockets.clients[subject] = { [id]: client }; + } else { + this.sockets.clients[subject][id] = client; + } + client.once("closed", () => { + const v = this.sockets.clients[subject]; + if (v != null) { + delete v[id]; + if (isEmpty(v)) { + delete this.sockets.clients[subject]; + } + } + }); + return client; + }, + }; + + private disconnectAllSockets = () => { + for (const subject in this.sockets.servers) { + this.sockets.servers[subject].disconnect(); + } + for (const subject in this.sockets.clients) { + for (const id in this.sockets.clients[subject]) { + this.sockets.clients[subject][id].disconnect(); + } + } + }; + + private closeAllSockets = () => { + for (const subject in this.sockets.servers) { + this.sockets.servers[subject].close(); + } + for (const subject in this.sockets.clients) { + for (const id in this.sockets.clients[subject]) { + this.sockets.clients[subject][id].close(); + } + } + }; + + message = (mesg, options?) => messageData(mesg, options); +} + +interface PublishOptions { + headers?: Headers; + // if encoding is given, it specifies the encoding used to encode the message + encoding?: DataEncoding; + // if raw is given, then it is assumed to be the raw binary + // encoded message (using encoding) and any mesg parameter + // is *IGNORED*. + raw?; + // timeout used when publishing a message and awaiting a response. + timeout?: number; +} + +interface RequestManyOptions extends PublishOptions { + maxWait?: number; + maxMessages?: number; +} + +export function encode({ + encoding, + mesg, +}: { + encoding: DataEncoding; + mesg: any; +}) { + if (encoding == DataEncoding.MsgPack) { + return msgpack.encode(mesg, MSGPACK_ENCODER_OPTIONS); + } else if (encoding == DataEncoding.JsonCodec) { + return jsonEncoder(mesg); + } else { + throw Error(`unknown encoding ${encoding}`); + } +} + +export function decode({ + encoding, + data, +}: { + encoding: DataEncoding; + data; +}): any { + if (encoding == DataEncoding.MsgPack) { + return msgpack.decode(data); + } else if (encoding == DataEncoding.JsonCodec) { + return jsonDecoder(data); + } else { + throw Error(`unknown encoding ${encoding}`); + } +} + +let textEncoder: TextEncoder | undefined = undefined; +let textDecoder: TextDecoder | undefined = undefined; + +function jsonEncoder(obj: any) { + if (textEncoder === undefined) { + textEncoder = new TextEncoder(); + } + return textEncoder.encode(JSON.stringify(obj)); +} + +function jsonDecoder(data: Buffer): any { + if (textDecoder === undefined) { + textDecoder = new TextDecoder(); + } + return JSON.parse(textDecoder.decode(data)); +} + +interface Chunk { + id: string; + seq: number; + done: number; + buffer: Buffer; + headers?: any; +} + +// if an incoming message has chunks at least this old +// we give up on it and discard all of them. This avoids +// memory leaks when a chunk is dropped. +const MAX_CHUNK_TIME = 2 * 60000; + +class SubscriptionEmitter extends EventEmitter { + private incoming: { [id: string]: (Partial & { time: number })[] } = + {}; + private client: Client; + private closeWhenOffCalled?: boolean; + private subject: string; + public refCount: number = 1; + + constructor({ client, subject, closeWhenOffCalled }) { + super(); + this.client = client; + this.subject = subject; + this.client.conn.on(subject, this.handle); + this.closeWhenOffCalled = closeWhenOffCalled; + this.dropOldLoop(); + } + + close = (force?) => { + this.refCount -= 1; + // console.log("SubscriptionEmitter.close - refCount =", this.refCount, this.subject); + if (this.client == null || (!force && this.refCount > 0)) { + return; + } + this.emit("closed"); + this.client.conn.removeListener(this.subject, this.handle); + // @ts-ignore + delete this.incoming; + // @ts-ignore + delete this.client; + // @ts-ignore + delete this.subject; + // @ts-ignore + delete this.closeWhenOffCalled; + this.removeAllListeners(); + }; + + off(a, b) { + super.off(a, b); + if (this.closeWhenOffCalled) { + this.close(); + } + return this; + } + + private handle = ({ subject, data }) => { + if (this.client == null) { + return; + } + const [id, seq, done, encoding, buffer, headers] = data; + // console.log({ id, seq, done, encoding, buffer, headers }); + const chunk = { seq, done, encoding, buffer, headers }; + const { incoming } = this; + if (incoming[id] == null) { + if (seq != 0) { + // part of a dropped message -- by definition this should just + // silently happen and be handled via application level encodings + // elsewhere + console.log( + `WARNING: drop packet from ${this.subject} -- first message has wrong seq`, + { seq }, + ); + return; + } + incoming[id] = []; + } else { + const prev = incoming[id].slice(-1)[0].seq ?? -1; + if (prev + 1 != seq) { + console.log( + `WARNING: drop packet from ${this.subject} -- seq number wrong`, + { prev, seq }, + ); + // part of message was dropped -- discard everything + delete incoming[id]; + return; + } + } + incoming[id].push({ ...chunk, time: Date.now() }); + if (chunk.done) { + // console.log("assembling ", incoming[id].length, "chunks"); + const chunks = incoming[id].map((x) => x.buffer!); + // TESTING ONLY!! + // This is not necessary due to the above checks as messages arrive. + // for (let i = 0; i < incoming[id].length; i++) { + // if (incoming[id][i]?.seq != i) { + // console.log(`WARNING: bug -- invalid chunk data! -- ${subject}`); + // throw Error("bug -- invalid chunk data!"); + // } + // } + const raw = concatArrayBuffers(chunks); + + // TESTING ONLY!! + // try { + // decode({ encoding, data: raw }); + // } catch (err) { + // console.log(`ERROR - invalid data ${subject}`, incoming[id], err); + // } + + delete incoming[id]; + const mesg = new Message({ + encoding, + raw, + headers, + client: this.client, + subject, + }); + this.emit("message", mesg); + this.client.recvStats(raw.byteLength); + } + }; + + dropOldLoop = async () => { + while (this.incoming != null) { + const cutoff = Date.now() - MAX_CHUNK_TIME; + for (const id in this.incoming) { + const chunks = this.incoming[id]; + if (chunks.length > 0 && chunks[0].time <= cutoff) { + console.log( + `WARNING: drop partial message from ${this.subject} due to timeout`, + ); + delete this.incoming[id]; + } + } + await delay(MAX_CHUNK_TIME / 2); + } + }; +} + +function concatArrayBuffers(buffers) { + if (buffers.length == 1) { + return buffers[0]; + } + if (Buffer.isBuffer(buffers[0])) { + return Buffer.concat(buffers); + } + // browser fallback + const totalLength = buffers.reduce((sum, buf) => sum + buf.byteLength, 0); + const result = new Uint8Array(totalLength); + let offset = 0; + for (const buf of buffers) { + result.set(new Uint8Array(buf), offset); + offset += buf.byteLength; + } + + return result.buffer; +} + +export type Headers = { [key: string]: JSONValue }; + +export class MessageData { + public readonly encoding: DataEncoding; + public readonly raw; + public readonly headers?: Headers; + + constructor({ encoding, raw, headers }) { + this.encoding = encoding; + this.raw = raw; + this.headers = headers; + } + + get data(): T { + return decode({ encoding: this.encoding, data: this.raw }); + } + + get length(): number { + // raw is binary data so it's the closest thing we have to the + // size of this message. It would also make sense to include + // the headers, but JSON'ing them would be expensive, so we don't. + return this.raw.length; + } +} + +export class Message extends MessageData { + private client: Client; + public readonly subject; + + constructor({ encoding, raw, headers, client, subject }) { + super({ encoding, raw, headers }); + this.client = client; + this.subject = subject; + } + + isRequest = (): boolean => !!this.headers?.[REPLY_HEADER]; + + private respondSubject = () => { + const subject = this.headers?.[REPLY_HEADER]; + if (!subject) { + console.log( + `WARNING: respond -- message to '${this.subject}' is not a request`, + ); + return; + } + return `${subject}`; + }; + + respondSync = (mesg, opts?: PublishOptions): { bytes: number } => { + const subject = this.respondSubject(); + if (!subject) return { bytes: 0 }; + return this.client.publishSync(subject, mesg, opts); + }; + + respond = async ( + mesg, + opts: PublishOptions = {}, + ): Promise<{ bytes: number; count: number }> => { + const subject = this.respondSubject(); + if (!subject) { + return { bytes: 0, count: 0 }; + } + return await this.client.publish(subject, mesg, opts); + }; +} + +export function messageData( + mesg, + { headers, raw, encoding = DEFAULT_ENCODING }: PublishOptions = {}, +) { + return new MessageData({ + encoding, + raw: raw ?? encode({ encoding, mesg }), + headers, + }); +} + +export type Subscription = EventIterator; + +export class ConatError extends Error { + code: string | number; + constructor(mesg: string, { code }) { + super(mesg); + this.code = code; + } +} + +function isEmpty(obj: object): boolean { + for (const _x in obj) { + return false; + } + return true; +} diff --git a/src/packages/conat/core/constants.ts b/src/packages/conat/core/constants.ts new file mode 100644 index 0000000000..f739208e31 --- /dev/null +++ b/src/packages/conat/core/constants.ts @@ -0,0 +1,16 @@ +// This is just the default with socket.io, but we might want a bigger +// size, which could mean more RAM usage by the servers. +// Our client protocol automatically chunks messages, so this payload +// size ONLY impacts performance, never application level constraints. +const MB = 1e6; +export const RESOURCE = "connections to CoCalc"; + +export const MAX_PAYLOAD = 8 * MB; + +export const MAX_SUBSCRIPTIONS_PER_CLIENT = 500; + +// hubs must have a much larger limit since they server everybody... +export const MAX_SUBSCRIPTIONS_PER_HUB = 15000; + +export const MAX_CONNECTIONS_PER_USER = 100; +export const MAX_CONNECTIONS = 10000; diff --git a/src/packages/conat/core/patterns.test.ts b/src/packages/conat/core/patterns.test.ts new file mode 100644 index 0000000000..ff70b032a9 --- /dev/null +++ b/src/packages/conat/core/patterns.test.ts @@ -0,0 +1,85 @@ +/* +DEVELOPMENT: + +pnpm test ./patterns.test.ts +*/ + +import { Patterns } from "./patterns"; +import { randomId } from "@cocalc/conat/names"; + +function expectEqual(actual: any[], expected: any[]) { + expect(actual).toEqual(expect.arrayContaining(expected)); + expect(actual).toHaveLength(expected.length); +} + +describe("test some basic pattern matching", () => { + it("tests some simple examples with just one or no matches", () => { + const p = new Patterns(); + p.set("x", 0); + p.set("a.b.>", 0); + p.set("a.*", 0); + expectEqual(p.matches("x"), ["x"]); + expectEqual(p.matches("y"), []); + expectEqual(p.matches("a.b.c"), ["a.b.>"]); + expectEqual(p.matches("a.b"), ["a.*"]); + }); + + it("some examples with several matches", () => { + const p = new Patterns(); + p.set("a.b.>", 0); + p.set("a.*.*", 0); + expectEqual(p.matches("a.b.c"), ["a.b.>", "a.*.*"]); + expectEqual(p.matches("a.b.c.d"), ["a.b.>"]); + }); + + it("example where we delete a pattern", () => { + const p = new Patterns(); + p.set("a.b.>", 0); + p.set("a.b.c", 0); + p.set("a.b.d", 0); + expectEqual(p.matches("a.b.c"), ["a.b.>", "a.b.c"]); + expectEqual(p.matches("a.b.d"), ["a.b.>", "a.b.d"]); + expectEqual(p.matches("a.b.c.d"), ["a.b.>"]); + p.delete("a.b.c"); + expectEqual(p.matches("a.b.d"), ["a.b.>", "a.b.d"]); + expectEqual(p.matches("a.b.c"), ["a.b.>"]); + p.delete("a.b.d"); + expectEqual(p.matches("a.b.d"), ["a.b.>"]); + p.delete("a.b.>"); + expectEqual(p.matches("a.b.d"), []); + }); +}); + +describe("do some stress tests", () => { + const patterns = 1e5; + + let p; + const knownIds: string[] = []; + it(`create ${patterns} patterns`, () => { + p = new Patterns(); + for (const seg1 of ["service", "hub", "project", "account", "global"]) { + for (const seg2 of ["*", "x"]) { + for (let i = 0; i < patterns / 10; i++) { + const id = randomId(); + knownIds.push(id); + const pattern = `${seg1}.${seg2}.${id}`; + p.set(pattern, 0); + } + } + } + }); + + const count = 1e6; + let m = 0; + it(`match ${count} times against them`, () => { + for (const seg1 of ["service", "hub", "project", "account", "global"]) { + for (const seg2 of ["a", "x"]) { + for (let i = 0; i < count / 10; i++) { + const subject = `${seg1}.${seg2}.${knownIds[i] ?? randomId()}`; + m = Math.max(p.matches(subject).length, m); + } + } + } + expect(m).toBeGreaterThan(0); + }); +}); diff --git a/src/packages/conat/core/patterns.ts b/src/packages/conat/core/patterns.ts new file mode 100644 index 0000000000..2c25f58624 --- /dev/null +++ b/src/packages/conat/core/patterns.ts @@ -0,0 +1,205 @@ +import { isEqual } from "lodash"; +import { getLogger } from "@cocalc/conat/client"; +import { EventEmitter } from "events"; + +type Index = { [pattern: string]: Index | string }; + +const logger = getLogger("pattern"); + +export class Patterns extends EventEmitter { + private patterns: { [pattern: string]: T } = {}; + private index: Index = {}; + + constructor() { + super(); + this.setMaxListeners(1000); + } + + close = () => { + this.emit("closed"); + this.patterns = {}; + this.index = {}; + }; + + serialize = (fromT?: (x: T) => any) => { + let patterns: { [pattern: string]: any }; + if (fromT != null) { + patterns = {}; + for (const pattern in this.patterns) { + patterns[pattern] = fromT(this.patterns[pattern]); + } + } else { + patterns = this.patterns; + } + + return { patterns, index: this.index }; + }; + + deserialize = ( + { patterns, index }: { patterns: { [pattern: string]: any }; index: Index }, + toT?: (x: any) => T, + ) => { + if (toT != null) { + for (const pattern in patterns) { + patterns[pattern] = toT(patterns[pattern]); // make it of type T + } + } + this.patterns = patterns; + this.index = index; + this.emit("change"); + }; + + // mutate this by merging in data from p. + merge = (p: Patterns) => { + for (const pattern in p.patterns) { + const t = p.patterns[pattern]; + this.set(pattern, t); + } + this.emit("change"); + }; + + matches = (subject: string): string[] => { + return matchUsingIndex(this.index, subject.split(".")); + }; + + matchesTest = (subject: string): string[] => { + const a = this.matches(subject); + const b = this.matchNaive(subject); + a.sort(); + b.sort(); + if (!isEqual(a, b)) { + logger.debug("BUG in PATTERN MATCHING!!!", { + subject, + a, + b, + index: this.index, + patterns: Object.keys(this.patterns), + }); + } + return b; + }; + + matchNaive = (subject: string): string[] => { + const v: string[] = []; + for (const pattern in this.patterns) { + if (matchesPattern(pattern, subject)) { + v.push(pattern); + } + } + return v; + }; + + get = (pattern: string): T | undefined => { + return this.patterns[pattern]; + }; + + set = (pattern: string, t: T) => { + this.patterns[pattern] = t; + setIndex(this.index, pattern.split("."), pattern); + this.emit("change"); + }; + + delete = (pattern: string) => { + delete this.patterns[pattern]; + deleteIndex(this.index, pattern.split(".")); + }; +} + +function setIndex(index: Index, segments: string[], pattern) { + if (segments.length == 0) { + index[""] = pattern; + return; + } + if (segments[0] == ">") { + // there can't be anything after it + index[">"] = pattern; + return; + } + const v = index[segments[0]]; + if (v === undefined) { + const idx: Index = {}; + setIndex(idx, segments.slice(1), pattern); + index[segments[0]] = idx; + return; + } + if (typeof v == "string") { + // already set + return; + } + setIndex(v, segments.slice(1), pattern); +} + +function deleteIndex(index: Index, segments: string[]) { + const ind = index[segments[0]]; + if (ind === undefined) { + return; + } + if (typeof ind != "string") { + deleteIndex(ind, segments.slice(1)); + // if there is anything still stored in ind + // besides ind[''], we do NOT delete it. + for (const key in ind) { + if (key != "") { + return; + } + } + } + delete index[segments[0]]; +} + +// todo deal with > +function matchUsingIndex(index: Index, segments: string[]): string[] { + if (segments.length == 0) { + const p = index[""]; + if (p === undefined) { + return []; + } else if (typeof p === "string") { + return [p]; + } else { + throw Error("bug"); + } + } + const matches: string[] = []; + const subject = segments[0]; + for (const pattern of ["*", ">", subject]) { + if (index[pattern] !== undefined) { + const p = index[pattern]; + if (typeof p == "string") { + // end of this pattern -- matches if segments also + // stops *or* this pattern is > + if (segments.length == 1) { + matches.push(p); + } else if (pattern == ">") { + matches.push(p); + } + } else { + for (const s of matchUsingIndex(p, segments.slice(1))) { + matches.push(s); + } + } + } + } + return matches; +} + +export function matchesSegment(pattern, subject): boolean { + if (pattern == "*" || pattern == ">") { + return true; + } + return pattern == subject; +} + +export function matchesPattern(pattern, subject): boolean { + const subParts = subject.split("."); + const patParts = pattern.split("."); + let i = 0, + j = 0; + while (i < subParts.length && j < patParts.length) { + if (patParts[j] === ">") return true; + if (patParts[j] !== "*" && patParts[j] !== subParts[i]) return false; + i++; + j++; + } + + return i === subParts.length && j === patParts.length; +} diff --git a/src/packages/conat/core/server.ts b/src/packages/conat/core/server.ts new file mode 100644 index 0000000000..50021548f0 --- /dev/null +++ b/src/packages/conat/core/server.ts @@ -0,0 +1,984 @@ +/* + +Just try it out, start up node.js in this directory and: + + s = require('@cocalc/conat/core/server').init({port:4567, getUser:()=>{return {hub_id:'hub'}}}) + c = s.client(); + c.watch('foo') + c2 = s.client(); + c2.pub('foo', 'bar') + + +cd packages/server + + + s = await require('@cocalc/server/conat/socketio').initConatServer() + + s0 = await require('@cocalc/server/conat/socketio').initConatServer({port:3000}); 0 + + +For valkey clustering -- run "valkey-server" in a terminal, then: + + s0 = await require('@cocalc/server/conat/socketio').initConatServer({valkey:'valkey://localhost:6379', port:3000, getUser:()=>{return {hub_id:'hub'}}}) + + s1 = await require('@cocalc/server/conat/socketio').initConatServer({valkey:'valkey://localhost:6379', port:3001, getUser:()=>{return {hub_id:'hub'}}}) + +Corresponding clients: + + c0 = require('@cocalc/conat/core/client').connect('http://localhost:3000') + + c1 = require('@cocalc/conat/core/client').connect('http://localhost:3001') + + +--- + +Or from cocalc/src + + pnpm conat-server + + +WARNING/TODO: I did not yet implement anything to expire interest +when a server terminates!! This basically isn't needed when using +the cluster adapter since there's no scaling up or down happening +(unless a worker keeps crashing), but with valkey it would be a good idea. + +*/ + +import type { ConnectionStats, ServerInfo } from "./types"; +import { + isValidSubject, + isValidSubjectWithoutWildcards, +} from "@cocalc/conat/util"; +import { createAdapter as createValkeyStreamsAdapter } from "@cocalc/redis-streams-adapter"; +import { createAdapter as createValkeyPubSubAdapter } from "@socket.io/redis-adapter"; +import Valkey from "iovalkey"; +import { Server } from "socket.io"; +import { callback, delay } from "awaiting"; +import { + ConatError, + connect, + type Client, + type ClientOptions, + MAX_INTEREST_TIMEOUT, + STICKY_QUEUE_GROUP, +} from "./client"; +import { + RESOURCE, + MAX_CONNECTIONS_PER_USER, + MAX_CONNECTIONS, + MAX_PAYLOAD, + MAX_SUBSCRIPTIONS_PER_CLIENT, + MAX_SUBSCRIPTIONS_PER_HUB, +} from "./constants"; +import { randomId } from "@cocalc/conat/names"; +import { Patterns } from "./patterns"; +import ConsistentHash from "consistent-hash"; +import { is_array } from "@cocalc/util/misc"; +import { UsageMonitor } from "@cocalc/conat/monitor/usage"; +import { once, until } from "@cocalc/util/async-utils"; +import { getLogger } from "@cocalc/conat/client"; + +const logger = getLogger("conat:core:server"); + +const INTEREST_STREAM = "interest"; +const STICKY_STREAM = "sticky"; + +const VALKEY_OPTIONS = { maxRetriesPerRequest: null }; +const USE_VALKEY_PUBSUB = true; + +const VALKEY_READ_COUNT = 100; + +export function valkeyClient(valkey) { + if (typeof valkey == "string") { + if (valkey.startsWith("{") && valkey.endsWith("}")) { + return new Valkey({ ...VALKEY_OPTIONS, ...JSON.parse(valkey) }); + } else { + return new Valkey(valkey, VALKEY_OPTIONS); + } + } else { + return new Valkey({ ...VALKEY_OPTIONS, ...valkey }); + } +} + +const DEBUG = false; + +interface InterestUpdate { + op: "add" | "delete"; + subject: string; + queue?: string; + room: string; +} + +interface StickyUpdate { + pattern: string; + subject: string; + target: string; +} + +export function init(opts: Options) { + return new ConatServer(opts); +} + +export type UserFunction = ( + socket, + systemAccounts?: { [cookieName: string]: { password: string; user: any } }, +) => Promise; + +export type AllowFunction = (opts: { + type: "pub" | "sub"; + user: any; + subject: string; +}) => Promise; + +export interface Options { + httpServer?; + port?: number; + id?: string; + path?: string; + getUser?: UserFunction; + isAllowed?: AllowFunction; + valkey?: + | string + | { + port?: number; + host?: string; + username?: string; + password?: string; + db?: number; + }; + cluster?: boolean; + maxSubscriptionsPerClient?: number; + maxSubscriptionsPerHub?: number; + systemAccountPassword?: string; + // if true, use https when creating an internal client. + ssl?: boolean; +} + +type State = "ready" | "closed"; + +export class ConatServer { + public readonly io; + public readonly id: string; + + private getUser: UserFunction; + private isAllowed: AllowFunction; + readonly options: Partial; + private cluster?: boolean; + + private sockets: { [id: string]: any } = {}; + private disconnectingTimeout: { + [id: string]: ReturnType; + } = {}; + + private stats: { [id: string]: ConnectionStats } = {}; + private usage: UsageMonitor; + private state: State = "ready"; + + private subscriptions: { [socketId: string]: Set } = {}; + private interest: Patterns<{ [queue: string]: Set }> = new Patterns(); + private sticky: { + // the target string is JSON.stringify({ id: string; subject: string }), which is the + // socket.io room to send the messages to. + [pattern: string]: { [subject: string]: string }; + } = {}; + + constructor(options: Options) { + const { + httpServer, + port = 3000, + ssl = false, + id = randomId(), + path = "/conat", + getUser, + isAllowed, + valkey, + cluster, + maxSubscriptionsPerClient = MAX_SUBSCRIPTIONS_PER_CLIENT, + maxSubscriptionsPerHub = MAX_SUBSCRIPTIONS_PER_HUB, + systemAccountPassword, + } = options; + this.options = { + port, + ssl, + id, + path, + valkey, + maxSubscriptionsPerClient, + maxSubscriptionsPerHub, + systemAccountPassword, + }; + this.cluster = cluster || !!valkey; + this.getUser = async (socket) => { + if (getUser == null) { + // no auth at all + return null; + } else { + let systemAccounts; + if (this.options.systemAccountPassword) { + systemAccounts = { + sys: { + password: this.options.systemAccountPassword, + user: { hub_id: "system" }, + }, + }; + } else { + systemAccounts = undefined; + } + return await getUser(socket, systemAccounts); + } + }; + this.isAllowed = isAllowed ?? (async () => true); + this.id = id; + this.log("Starting Conat server...", { + id, + path, + port: this.options.port, + httpServer: httpServer ? "httpServer(...)" : undefined, + valkey: !!valkey, // valkey has password in it so do not log + }); + + // NOTE: do NOT enable connectionStateRecovery; it seems to cause issues + // when restarting the server. + let adapter: any = undefined; + if (valkey) { + this.log("using valkey"); + const c = valkeyClient(valkey); + if (USE_VALKEY_PUBSUB) { + this.log("using the valkey pub/sub adapter"); + adapter = createValkeyPubSubAdapter(c, c.duplicate()); + } else { + this.log("using the valkey streams adapter with low-latency config"); + adapter = createValkeyStreamsAdapter(c, { + readCount: VALKEY_READ_COUNT, + blockTime: 1, + }); + } + } + + const socketioOptions = { + maxHttpBufferSize: MAX_PAYLOAD, + path, + adapter, + // perMessageDeflate is disabled by default in socket.io, but it + // seems unclear exactly *why*: + // https://github.com/socketio/socket.io/issues/3477#issuecomment-930503313 + perMessageDeflate: { threshold: 1024 }, + }; + this.log(socketioOptions); + if (httpServer) { + this.io = new Server(httpServer, socketioOptions); + } else { + this.io = new Server(port, socketioOptions); + this.log(`listening on port ${port}`); + } + this.initUsage(); + this.init(); + if (this.options.systemAccountPassword) { + this.initSystemService(); + } + } + + private init = async () => { + this.io.on("connection", this.handleSocket); + if (this.cluster) { + if (this.options.valkey == null) { + // the cluster adapter doesn't get configured until after the constructor, + // so we wait a moment before configuring these. + await delay(1); + } + this.initInterestSubscription(); + this.initStickySubscription(); + } + }; + + private initUsage = () => { + this.usage = new UsageMonitor({ + maxPerUser: MAX_CONNECTIONS_PER_USER, + max: MAX_CONNECTIONS, + resource: RESOURCE, + log: (...args) => this.log("usage", ...args), + }); + }; + + close = async () => { + if (this.state == "closed") { + return; + } + this.state = "closed"; + await this.io.close(); + for (const prop of ["interest", "subscriptions", "sockets", "services"]) { + delete this[prop]; + } + this.usage?.close(); + this.interest?.close(); + this.sticky = {}; + this.subscriptions = {}; + this.stats = {}; + this.sockets = {}; + }; + + private info = (): ServerInfo => { + return { + max_payload: MAX_PAYLOAD, + id: this.id, + }; + }; + + private log = (...args) => { + logger.debug(this.id, ":", ...args); + }; + + private unsubscribe = async ({ socket, subject }) => { + if (DEBUG) { + this.log("unsubscribe ", { id: socket.id, subject }); + } + const room = socketSubjectRoom({ socket, subject }); + socket.leave(room); + await this.updateInterest({ op: "delete", subject, room }); + }; + + // INTEREST + + private updateInterest = async (update: InterestUpdate) => { + this._updateInterest(update); + if (!this.cluster) return; + // console.log(this.options.port, "cluster: publish interest change", update); + this.io.of("cluster").serverSideEmit(INTEREST_STREAM, "update", update); + }; + + private initInterest = async () => { + if (!this.cluster) return; + const getStateFromCluster = (cb) => { + this.io.of("cluster").serverSideEmit(INTEREST_STREAM, "init", cb); + }; + + await until( + async () => { + try { + const responses = (await callback(getStateFromCluster)).filter( + (state) => isNonempty(state.patterns), + ); + // console.log("initInterest got", responses); + if (responses.length > 0) { + this.deserializeInterest(responses[0]); + return true; + } else { + // console.log(`init interest state -- waiting for other nodes...`); + return false; + } + } catch (err) { + console.log(`initInterest: WARNING -- ${err}`); + return false; + } + }, + { start: 100, decay: 1.5, max: 5000 }, + ); + }; + + private initInterestSubscription = async () => { + if (!this.cluster) return; + + this.initInterest(); + + this.io.of("cluster").on(INTEREST_STREAM, (action, args) => { + // console.log("INTEREST_STREAM received", { action, args }); + if (action == "update") { + // another server telling us about subscription interest + // console.log("applying interest update", args); + this._updateInterest(args); + } else if (action == "init") { + // console.log("another server requesting state"); + args(this.serializableInterest()); + } + }); + }; + + private serializableInterest = () => { + const fromT = (x: { [queue: string]: Set }) => { + const y: { [queue: string]: string[] } = {}; + for (const queue in x) { + y[queue] = Array.from(x[queue]); + } + return y; + }; + return this.interest.serialize(fromT); + }; + + private deserializeInterest = (state) => { + const interest = new Patterns<{ [queue: string]: Set }>(); + interest.deserialize(state, (x: any) => { + for (const key in x) { + x[key] = new Set(x[key]); + } + return x; + }); + const i = this.interest; + this.interest = interest; + this.interest.merge(i); + }; + + private _updateInterest = (update: InterestUpdate) => { + if (this.state != "ready") return; + const { op, subject, queue, room } = update; + const groups = this.interest.get(subject); + if (op == "add") { + if (typeof queue != "string") { + throw Error("queue must not be null for add"); + } + if (groups === undefined) { + this.interest.set(subject, { [queue]: new Set([room]) }); + } else if (groups[queue] == null) { + groups[queue] = new Set([room]); + } else { + groups[queue].add(room); + } + } else if (op == "delete") { + if (groups != null) { + let nonempty = false; + for (const queue in groups) { + groups[queue].delete(room); + if (groups[queue].size == 0) { + delete groups[queue]; + } else { + nonempty = true; + } + } + if (!nonempty) { + // no interest anymore + this.interest.delete(subject); + delete this.sticky[subject]; + } + } + } else { + throw Error(`invalid op ${op}`); + } + }; + + // STICKY + + private initSticky = async () => { + if (!this.cluster) return; + const getStateFromCluster = (cb) => { + this.io.of("cluster").serverSideEmit(STICKY_STREAM, "init", cb); + }; + + await until( + async () => { + try { + const responses = (await callback(getStateFromCluster)).filter((x) => + isNonempty(x), + ); + // console.log("initSticky got", responses); + if (responses.length > 0) { + for (const response of responses) { + this.mergeSticky(response); + } + return true; + } else { + // console.log(`init sticky state -- waiting for other nodes...`); + return false; + } + } catch (err) { + console.log(`initInterest: WARNING -- ${err}`); + return false; + } + }, + { start: 100, decay: 1.5, max: 10000 }, + ); + }; + + private mergeSticky = (sticky: { + [pattern: string]: { [subject: string]: string }; + }) => { + for (const pattern in sticky) { + this.sticky[pattern] = { ...sticky[pattern], ...this.sticky[pattern] }; + } + }; + + private initStickySubscription = async () => { + if (!this.cluster) return; + + this.initSticky(); + + this.io.of("cluster").on(STICKY_STREAM, (action, args) => { + // console.log("STICKY_STREAM received", { action, args }); + if (action == "update") { + this._updateSticky(args); + } else if (action == "init") { + // console.log("sending stickyUpdates", this.stickyUpdates); + args(this.sticky); + } + }); + }; + + private updateSticky = async (update: StickyUpdate) => { + this._updateSticky(update); + if (!this.cluster) return; + + // console.log(this.options.port, "cluster: publish sticky update", update); + this.io.of("cluster").serverSideEmit(STICKY_STREAM, "update", update); + }; + + private _updateSticky = (update: StickyUpdate) => { + const { pattern, subject, target } = update; + if (this.sticky[pattern] === undefined) { + this.sticky[pattern] = {}; + } + this.sticky[pattern][subject] = target; + }; + + private getStickyTarget = ({ pattern, subject }) => { + return this.sticky[pattern]?.[subject]; + }; + + // + + private subscribe = async ({ socket, subject, queue, user }) => { + if (DEBUG) { + this.log("subscribe ", { id: socket.id, subject, queue }); + } + if (typeof queue != "string") { + throw Error("queue must be defined"); + } + if (!isValidSubject(subject)) { + throw Error("invalid subject"); + return; + } + if (!(await this.isAllowed({ user, subject, type: "sub" }))) { + const message = `permission denied subscribing to '${subject}' from ${JSON.stringify(user)}`; + this.log(message); + throw new ConatError(message, { + code: 403, + }); + } + let maxSubs; + if (user?.hub_id) { + maxSubs = + this.options.maxSubscriptionsPerHub ?? MAX_SUBSCRIPTIONS_PER_HUB; + } else { + maxSubs = + this.options.maxSubscriptionsPerClient ?? MAX_SUBSCRIPTIONS_PER_CLIENT; + } + if (maxSubs) { + const numSubs = this.subscriptions?.[socket.id]?.size ?? 0; + if (numSubs >= maxSubs) { + // error 429 == "too many requests" + throw new ConatError( + `there is a limit of at most ${maxSubs} subscriptions and you currently have ${numSubs} subscriptions -- subscription to '${subject}' denied`, + { code: 429 }, + ); + } + } + const room = socketSubjectRoom({ socket, subject }); + // critical to await socket.join so we don't advertise that there is + // a subscriber before the socket is actually getting messages. + await socket.join(room); + await this.updateInterest({ op: "add", subject, room, queue }); + }; + + private publish = async ({ subject, data, from }): Promise => { + if (!isValidSubjectWithoutWildcards(subject)) { + throw Error("invalid subject"); + } + if (!(await this.isAllowed({ user: from, subject, type: "pub" }))) { + const message = `permission denied publishing to '${subject}' from ${JSON.stringify(from)}`; + this.log(message); + throw new ConatError(message, { + // this is the http code for permission denied, and having this + // set is assumed elsewhere in our code, so don't mess with it! + code: 403, + }); + } + let count = 0; + for (const pattern of this.interest.matches(subject)) { + const g = this.interest.get(pattern)!; + if (DEBUG) { + this.log("publishing", { subject, data, g }); + } + // send to exactly one in each queue group + for (const queue in g) { + const target = this.loadBalance({ + pattern, + subject, + queue, + targets: g[queue], + }); + if (target !== undefined) { + this.io.to(target).emit(pattern, { subject, data }); + count += 1; + } + } + } + return count; + }; + + private loadBalance = ({ + pattern, + subject, + queue, + targets, + }: { + pattern: string; + subject: string; + queue: string; + targets: Set; + }): string | undefined => { + if (targets.size == 0) { + return undefined; + } + if (queue == STICKY_QUEUE_GROUP) { + const v = subject.split("."); + subject = v.slice(0, v.length - 1).join("."); + const currentTarget = this.getStickyTarget({ pattern, subject }); + if (currentTarget === undefined || !targets.has(currentTarget)) { + // we use consistent hashing instead of random to make the choice, because if + // choice is being made by two different socketio servers at the same time, + // and they make different choices, it would be (temporarily) bad since a + // couple messages could get routed inconsistently (valkey sync would quickly + // resolve this). It's actually very highly likely to have such parallel choices + // happening in cocalc, since when a file is opened a persistent stream is opened + // in the browser and the project at the exact same time, and those are likely + // to be connected to different socketio servers. By using consistent hashing, + // all conflicts are avoided except for a few moments when the actual targets + // (e.g., the persist servers) are themselves changing, which should be something + // that only happens for a moment every few days. + const target = consistentChoice(targets, subject); + this.updateSticky({ pattern, subject, target }); + return target; + } + return currentTarget; + } else { + return randomChoice(targets); + } + }; + + private handleSocket = async (socket) => { + this.sockets[socket.id] = socket; + socket.once("closed", () => { + delete this.sockets[socket.id]; + delete this.stats[socket.id]; + }); + + this.stats[socket.id] = { + send: { messages: 0, bytes: 0 }, + recv: { messages: 0, bytes: 0 }, + subs: 0, + connected: Date.now(), + address: getAddress(socket), + }; + let user: any = null; + let added = false; + try { + user = await this.getUser(socket); + this.usage.add(user); + added = true; + } catch (err) { + // getUser is supposed to throw an error if authentication fails + // for any reason + // Also, if the connection limit is hit they still connect, but as + // the error user who can't do anything (hence not waste resources). + user = { error: `${err}`, code: err.code }; + } + this.stats[socket.id].user = user; + const id = socket.id; + this.log("new connection", { id, user }); + if (this.disconnectingTimeout[id]) { + this.log("clearing disconnectingTimeout - ", { id, user }); + clearTimeout(this.disconnectingTimeout[id]); + delete this.disconnectingTimeout[id]; + } + if (this.subscriptions[id] == null) { + this.subscriptions[id] = new Set(); + } + + socket.emit("info", { ...this.info(), user }); + + socket.on("stats", ({ recv0 }) => { + const s = this.stats[socket.id]; + if (s == null) return; + s.recv = recv0; + }); + + socket.on( + "wait-for-interest", + async ({ subject, timeout = MAX_INTEREST_TIMEOUT }, respond) => { + if (respond == null) { + return; + } + if (!isValidSubjectWithoutWildcards(subject)) { + respond({ error: "invalid subject" }); + return; + } + if (!(await this.isAllowed({ user, subject, type: "pub" }))) { + const message = `permission denied waiting for interest in '${subject}' from ${JSON.stringify(user)}`; + this.log(message); + respond({ error: message, code: 403 }); + } + const matches = this.interest.matches(subject); + if (matches.length > 0 || !timeout) { + // NOTE: we never return the actual matches, since this is a potential security vulnerability. + // it could make it very easy to figure out private inboxes, etc. + respond(matches.length > 0); + } + if (timeout > MAX_INTEREST_TIMEOUT) { + timeout = MAX_INTEREST_TIMEOUT; + } + const start = Date.now(); + while (this.state != "closed" && this.sockets[socket.id]) { + if (Date.now() - start >= timeout) { + respond({ error: "timeout" }); + return; + } + await once(this.interest, "change"); + if ((this.state as any) == "closed" || !this.sockets[socket.id]) { + return; + } + const matches = this.interest.matches(subject); + if (matches.length > 0) { + respond(true); + return; + } + } + }, + ); + + socket.on("publish", async ([subject, ...data], respond) => { + if (data?.[2]) { + // done + this.stats[socket.id].send.messages += 1; + } + this.stats[socket.id].send.bytes += data[4]?.length ?? 0; + this.stats[socket.id].active = Date.now(); + // this.log(JSON.stringify(this.stats)); + + try { + const count = await this.publish({ subject, data, from: user }); + respond?.({ count }); + } catch (err) { + if (err.code == 403) { + socket.emit("permission", { + message: err.message, + subject, + type: "pub", + }); + } + respond?.({ error: `${err}`, code: err.code }); + } + }); + + const subscribe = async ({ subject, queue }) => { + try { + if (this.subscriptions[id].has(subject)) { + return { status: "already-added" }; + } + await this.subscribe({ socket, subject, queue, user }); + this.subscriptions[id].add(subject); + this.stats[socket.id].subs += 1; + this.stats[socket.id].active = Date.now(); + return { status: "added" }; + } catch (err) { + if (err.code == 403) { + socket.emit("permission", { + message: err.message, + subject, + type: "sub", + }); + } + return { error: `${err}`, code: err.code }; + } + }; + + socket.on( + "subscribe", + async (x: { subject; queue } | { subject; queue }[], respond) => { + let r; + if (is_array(x)) { + const v: any[] = []; + for (const y of x) { + v.push(await subscribe(y)); + } + r = v; + } else { + r = await subscribe(x); + } + respond?.(r); + }, + ); + + socket.on("subscriptions", (_, respond) => { + if (respond == null) { + return; + } + respond(Array.from(this.subscriptions[id])); + }); + + const unsubscribe = ({ subject }: { subject: string }) => { + if (!this.subscriptions[id].has(subject)) { + return; + } + this.unsubscribe({ socket, subject }); + this.subscriptions[id].delete(subject); + this.stats[socket.id].subs -= 1; + this.stats[socket.id].active = Date.now(); + }; + + socket.on( + "unsubscribe", + (x: { subject: string } | { subject: string }[], respond) => { + let r; + if (is_array(x)) { + r = x.map(unsubscribe); + } else { + r = unsubscribe(x); + } + respond?.(r); + }, + ); + + socket.on("disconnecting", async () => { + this.log("disconnecting", { id, user }); + delete this.stats[socket.id]; + if (added) { + this.usage.delete(user); + } + const rooms = Array.from(socket.rooms) as string[]; + for (const room of rooms) { + const subject = getSubjectFromRoom(room); + this.unsubscribe({ socket, subject }); + } + delete this.subscriptions[id]; + }); + }; + + // create new client in the same process connected to this server. + // This is useful for unit testing and is not cached by default (i.e., multiple + // calls return distinct clients). + private address = () => { + const port = this.options.port; + const path = this.options.path?.slice(0, -"/conat".length) ?? ""; + return `http${this.options.ssl || port == 443 ? "s" : ""}://localhost:${port}${path}`; + }; + + client = (options?: ClientOptions): Client => { + const address = this.address(); + this.log("client: connecting to - ", { address }); + return connect({ + address, + noCache: true, + ...options, + }); + }; + + initSystemService = async () => { + if (!this.options.systemAccountPassword) { + throw Error("system service requires system account"); + } + this.log("starting service listening on sys..."); + const client = this.client({ + extraHeaders: { Cookie: `sys=${this.options.systemAccountPassword}` }, + }); + try { + await client.service( + "sys.conat.server", + { + stats: () => { + return { [this.id]: this.stats }; + }, + usage: () => { + return { [this.id]: this.usage.stats() }; + }, + // user has to explicitly refresh there browser after + // being disconnected this way + disconnect: (ids: string | string[]) => { + if (typeof ids == "string") { + ids = [ids]; + } + for (const id of ids) { + this.io.in(id).disconnectSockets(); + } + }, + }, + { queue: this.id }, + ); + this.log(`successfully started sys.conat.server service`); + } catch (err) { + this.log(`WARNING: unable to start sys.conat.server service -- ${err}`); + } + }; +} + +function getSubjectFromRoom(room: string) { + if (room.startsWith("{")) { + return JSON.parse(room).subject; + } else { + return room; + } +} + +function socketSubjectRoom({ socket, subject }) { + return JSON.stringify({ id: socket.id, subject }); +} + +export function randomChoice(v: Set): string { + if (v.size == 0) { + throw Error("v must have size at least 1"); + } + if (v.size == 1) { + for (const x of v) { + return x; + } + } + const w = Array.from(v); + const i = Math.floor(Math.random() * w.length); + return w[i]; +} + +export function consistentChoice(v: Set, resource: string): string { + if (v.size == 0) { + throw Error("v must have size at least 1"); + } + if (v.size == 1) { + for (const x of v) { + return x; + } + } + const hr = new ConsistentHash(); + const w = Array.from(v); + w.sort(); + for (const x of w) { + hr.add(x); + } + return hr.get(resource); +} + +// See https://socket.io/how-to/get-the-ip-address-of-the-client +function getAddress(socket) { + const header = socket.handshake.headers["forwarded"]; + if (header) { + for (const directive of header.split(",")[0].split(";")) { + if (directive.startsWith("for=")) { + return directive.substring(4); + } + } + } + + let addr = socket.handshake.headers["x-forwarded-for"]?.split(",")?.[0]; + if (addr) { + return addr; + } + for (const other of ["cf-connecting-ip", "fastly-client-ip"]) { + addr = socket.handshake.headers[other]; + if (addr) { + return addr; + } + } + + return socket.handshake.address; +} + +function isNonempty(obj) { + for (const _ in obj) { + return true; + } + return false; +} diff --git a/src/packages/conat/core/types.ts b/src/packages/conat/core/types.ts new file mode 100644 index 0000000000..615c3760c4 --- /dev/null +++ b/src/packages/conat/core/types.ts @@ -0,0 +1,23 @@ +interface User { + account_id?: string; + project_id?: string; + hub_id?: string; + error?: string; +} + +export interface ServerInfo { + max_payload: number; + id: string; + user?: User; +} + +export interface ConnectionStats { + user?: User; + send: { messages: number; bytes: number }; + recv: { messages: number; bytes: number }; + subs: number; + connected?: number; // time connected + active?: number; + // ip address + address?: string; +} diff --git a/src/packages/nats/files/read.ts b/src/packages/conat/files/read.ts similarity index 59% rename from src/packages/nats/files/read.ts rename to src/packages/conat/files/read.ts index 619fbd6a36..d32be0eb4e 100644 --- a/src/packages/nats/files/read.ts +++ b/src/packages/conat/files/read.ts @@ -2,7 +2,7 @@ Read a file from a project/compute server via an async generator, so it is memory efficient. -This is a NATS service that uses requestMany, takes as input a filename path, and streams all +This is a conat service that uses requestMany, takes as input a filename path, and streams all the binary data from that path. We use headers to add sequence numbers into the response messages. @@ -24,13 +24,13 @@ over a websocket for compute servers, so would just copy that code. DEVELOPMENT: -See src/packages/backend/nats/test/files/read.test.ts for unit tests. +See src/packages/backend/conat/test/files/read.test.ts for unit tests. ~/cocalc/src/packages/backend$ node -require('@cocalc/backend/nats'); a = require('@cocalc/nats/files/read'); a.createServer({project_id:'00847397-d6a8-4cb0-96a8-6ef64ac3e6cf',compute_server_id:0,createReadStream:require('fs').createReadStream}) +require('@cocalc/backend/conat'); a = require('@cocalc/conat/files/read'); a.createServer({project_id:'00847397-d6a8-4cb0-96a8-6ef64ac3e6cf',compute_server_id:0,createReadStream:require('fs').createReadStream}) -for await (const chunk of await a.readFile({project_id:'00847397-d6a8-4cb0-96a8-6ef64ac3e6cf',compute_server_id:0,path:'/tmp/a.py'})) { console.log({chunk}); } +for await (const chunk of await a.readFile({project_id:'00847397-d6a8-4cb0-96a8-6ef64ac3e6cf',compute_server_id:0,path:'/tmp/a'})) { console.log({chunk}); } for await (const chunk of await a.readFile({project_id:'00847397-d6a8-4cb0-96a8-6ef64ac3e6cf',compute_server_id:0,path:'/projects/6b851643-360e-435e-b87e-f9a6ab64a8b1/cocalc/.git/objects/pack/pack-771f7fe4ee855601463be070cf9fb9afc91f84ac.pack'})) { console.log({chunk}); } @@ -38,19 +38,18 @@ for await (const chunk of await a.readFile({project_id:'00847397-d6a8-4cb0-96a8- */ -import { getEnv } from "@cocalc/nats/client"; -import { projectSubject } from "@cocalc/nats/names"; -import { Empty, headers, type Subscription } from "@nats-io/nats-core"; -import { runLoop } from "./util"; +import { conat } from "@cocalc/conat/client"; +import { projectSubject } from "@cocalc/conat/names"; +import { type Subscription } from "@cocalc/conat/core/client"; let subs: { [name: string]: Subscription } = {}; export async function close({ project_id, compute_server_id, name = "" }) { - const key = getSubject({ project_id, compute_server_id, name }); - if (subs[key] == null) { + const subject = getSubject({ project_id, compute_server_id, name }); + if (subs[subject] == null) { return; } - const sub = subs[key]; - delete subs[key]; + const sub = subs[subject]; + delete subs[subject]; await sub.drain(); } @@ -73,17 +72,10 @@ export async function createServer({ compute_server_id, name, }); - if (subs[subject] != null) { - return; - } - const { nc } = await getEnv(); - runLoop({ - listen, - subs, - subject, - nc, - opts: { createReadStream }, - }); + const cn = await conat(); + const sub = await cn.subscribe(subject); + subs[subject] = sub; + listen({ sub, createReadStream }); } async function listen({ sub, createReadStream }) { @@ -99,40 +91,32 @@ async function listen({ sub, createReadStream }) { async function handleMessage(mesg, createReadStream) { try { await sendData(mesg, createReadStream); - const h = headers(); - h.append("done", ""); - mesg.respond(Empty, { headers: h }); + await mesg.respond(null, { headers: { done: true } }); } catch (err) { - const h = headers(); - h.append("error", `${err}`); // console.log("sending ERROR", err); - mesg.respond(Empty, { headers: h }); + mesg.respondSync(null, { headers: { error: `${err}` } }); } } -const MAX_NATS_CHUNK_SIZE = 16384 * 16 * 3; +const MAX_CHUNK_SIZE = 16384 * 16 * 3; function getSeqHeader(seq) { - const h = headers(); - h.append("seq", `${seq}`); - return { headers: h }; + return { headers: { seq } }; } async function sendData(mesg, createReadStream) { - const { jc } = await getEnv(); - const { path } = jc.decode(mesg.data); + const { path } = mesg.data; let seq = 0; for await (let chunk of createReadStream(path, { highWaterMark: 16384 * 16 * 3, })) { // console.log("sending ", { seq, bytes: chunk.length }); // We must break the chunk into smaller messages or it will - // get bounced by nats... TODO: can we get the max - // message size from nats? + // get bounced by conat... while (chunk.length > 0) { seq += 1; - mesg.respond(chunk.slice(0, MAX_NATS_CHUNK_SIZE), getSeqHeader(seq)); - chunk = chunk.slice(MAX_NATS_CHUNK_SIZE); + mesg.respondSync(chunk.slice(0, MAX_CHUNK_SIZE), getSeqHeader(seq)); + chunk = chunk.slice(MAX_CHUNK_SIZE); } } } @@ -152,7 +136,7 @@ export async function* readFile({ name = "", maxWait = 1000 * 60 * 10, // 10 minutes }: ReadFileOptions) { - const { nc, jc } = await getEnv(); + const cn = await conat(); const subject = getSubject({ project_id, compute_server_id, @@ -161,23 +145,30 @@ export async function* readFile({ const v: any = []; let seq = 0; let bytes = 0; - for await (const resp of await nc.requestMany(subject, jc.encode({ path }), { - maxWait, - })) { - for (const [key, value] of resp.headers ?? []) { - if (key == "error") { - throw Error(value[0] ?? "bug"); - } else if (key == "done") { - return; - } else if (key == "seq") { - const next = parseInt(value[0]); - bytes = resp.data.length; - // console.log("received seq", { seq: next, bytes }); - if (next != seq + 1) { - throw Error(`lost data: seq=${seq}, next=${next}`); - } - seq = next; + for await (const resp of await cn.requestMany( + subject, + { path }, + { + maxWait, + }, + )) { + if (resp.headers == null) { + continue; + } + if (resp.headers.error) { + throw Error(`${resp.headers.error}`); + } + if (resp.headers.done) { + return; + } + if (resp.headers.seq) { + const next = resp.headers.seq as number; + bytes = resp.data.length; + // console.log("received seq", { seq: next, bytes }); + if (next != seq + 1) { + throw Error(`lost data: seq=${seq}, next=${next}`); } + seq = next; } yield resp.data; } diff --git a/src/packages/nats/files/write.ts b/src/packages/conat/files/write.ts similarity index 78% rename from src/packages/nats/files/write.ts rename to src/packages/conat/files/write.ts index f3109d1774..6681fbcd8f 100644 --- a/src/packages/nats/files/write.ts +++ b/src/packages/conat/files/write.ts @@ -1,16 +1,16 @@ /* -Streaming write over NATS to a project or compute server. +Streaming write over Conat to a project or compute server. This is a key component to support user uploads, while being memory efficient -by streaming the write. Basically it uses NATS to support efficiently doing +by streaming the write. Basically it uses conat to support efficiently doing streaming writes of files to any compute server or project that is somehow -connected to NATS. +connected to conat. INSTRUCTIONS: Import writeFile: - import { writeFile } from "@cocalc/nats/files/write"; + import { writeFile } from "@cocalc/conat/files/write"; Now you can write a given path to a project (or compute_server) as simply as this: @@ -30,7 +30,7 @@ HOW THIS WORKS: Here's how this works from the side of the compute server: -- We start a request/response NATS server on the compute server: +- We start a request/response conat server on the compute server: - There's one message it accepts, which is: "Using streaming download to get {path} from [subject]." The sender of that message should set a long timeout (e.g., 10 minutes). @@ -50,11 +50,11 @@ Here's how it works from the side of whoever is sending the file: DEVELOPMENT: -See src/packages/backend/nats/test/files/write.test.ts for unit tests. +See src/packages/backend/conat/test/files/write.test.ts for unit tests. ~/cocalc/src/packages/backend$ node -require('@cocalc/backend/nats'); a = require('@cocalc/nats/files/write'); +require('@cocalc/backend/conat'); a = require('@cocalc/conat/files/write'); project_id = '00847397-d6a8-4cb0-96a8-6ef64ac3e6cf'; compute_server_id = 0; await a.createServer({project_id,compute_server_id,createWriteStream:require('fs').createWriteStream}); @@ -63,17 +63,16 @@ await a.writeFile({stream, project_id, compute_server_id, path:'/tmp/a.ts'}) */ -import { getEnv } from "@cocalc/nats/client"; -import { readFile } from "./read"; -import { randomId } from "@cocalc/nats/names"; +import { conat } from "@cocalc/conat/client"; +import { randomId } from "@cocalc/conat/names"; import { close as closeReadService, createServer as createReadServer, + readFile, } from "./read"; -import { projectSubject } from "@cocalc/nats/names"; -import { type Subscription } from "@nats-io/nats-core"; +import { projectSubject } from "@cocalc/conat/names"; +import { type Subscription } from "@cocalc/conat/core/client"; import { type Readable } from "node:stream"; -import { runLoop } from "./util"; function getWriteSubject({ project_id, compute_server_id }) { return projectSubject({ @@ -85,12 +84,12 @@ function getWriteSubject({ project_id, compute_server_id }) { let subs: { [name: string]: Subscription } = {}; export async function close({ project_id, compute_server_id }) { - const key = getWriteSubject({ project_id, compute_server_id }); - if (subs[key] == null) { + const subject = getWriteSubject({ project_id, compute_server_id }); + if (subs[subject] == null) { return; } - const sub = subs[key]; - delete subs[key]; + const sub = subs[subject]; + delete subs[subject]; await sub.drain(); } @@ -111,14 +110,10 @@ export async function createServer({ if (sub != null) { return; } - const { nc } = await getEnv(); - runLoop({ - listen, - subs, - subject, - nc, - opts: { createWriteStream, project_id, compute_server_id }, - }); + const cn = await conat(); + sub = await cn.subscribe(subject); + subs[subject] = sub; + listen({ sub, createWriteStream, project_id, compute_server_id }); } async function listen({ @@ -143,15 +138,14 @@ async function handleMessage({ compute_server_id, }) { let error = ""; - const { jc } = await getEnv(); let writeStream: null | Awaited> = null; try { - const { path, name, maxWait } = jc.decode(mesg.data); + const { path, name, maxWait } = mesg.data; writeStream = await createWriteStream(path); // console.log("created writeStream"); writeStream.on("error", (err) => { error = `${err}`; - mesg.respond(jc.encode({ error, status: "error" })); + mesg.respondSync({ error, status: "error" }); console.warn(`error writing ${path}: ${error}`); writeStream.emit("remove"); }); @@ -176,10 +170,10 @@ async function handleMessage({ } writeStream.end(); writeStream.emit("rename"); - mesg.respond(jc.encode({ status: "success", bytes, chunks })); + mesg.respondSync({ status: "success", bytes, chunks }); } catch (err) { if (!error) { - mesg.respond(jc.encode({ error: `${err}`, status: "error" })); + mesg.respondSync({ error: `${err}`, status: "error" }); writeStream?.emit("remove"); } } @@ -213,13 +207,13 @@ export async function writeFile({ name, }); // tell compute server to start reading our file. - const { nc, jc } = await getEnv(); - const resp = await nc.request( + const cn = await conat(); + const resp = await cn.request( getWriteSubject({ project_id, compute_server_id }), - jc.encode({ name, path, maxWait }), + { name, path, maxWait }, { timeout: maxWait }, ); - const { error, bytes, chunks } = jc.decode(resp.data); + const { error, bytes, chunks } = resp.data; if (error) { throw Error(error); } diff --git a/src/packages/nats/hub-api/db.ts b/src/packages/conat/hub/api/db.ts similarity index 90% rename from src/packages/nats/hub-api/db.ts rename to src/packages/conat/hub/api/db.ts index b3e94ceb19..8b322b003e 100644 --- a/src/packages/nats/hub-api/db.ts +++ b/src/packages/conat/hub/api/db.ts @@ -1,4 +1,4 @@ -import { authFirst } from "./util"; +import { authFirst, requireAccount } from "./util"; export const db = { userQuery: authFirst, @@ -6,6 +6,7 @@ export const db = { getLegacyTimeTravelInfo: authFirst, getLegacyTimeTravelPatches: authFirst, fileUseTimes: authFirst, + removeBlobTtls: requireAccount, }; export interface DB { @@ -33,13 +34,13 @@ export interface DB { getLegacyTimeTravelPatches: (opts: { account_id?: string; uuid: string; - // you should set this to true to enable potentially very large response support - requestMany?: boolean; // also, make this bigger: timeout?: number; }) => Promise; fileUseTimes: (opts: FileUseTimesOptions) => Promise; + + removeBlobTtls: (opts: { uuids: string[] }) => Promise; } export interface FileUseTimesOptions { diff --git a/src/packages/nats/hub-api/index.ts b/src/packages/conat/hub/api/index.ts similarity index 79% rename from src/packages/nats/hub-api/index.ts rename to src/packages/conat/hub/api/index.ts index 323846b9f7..f8a22fef54 100644 --- a/src/packages/nats/hub-api/index.ts +++ b/src/packages/conat/hub/api/index.ts @@ -1,23 +1,17 @@ -/* -NOTE: If you need to send *very large responses* to a message or increase timeouts, -see getLegacyTimeTravelPatches in db.ts. You just have to allow the keys requestMany -and timeout to the *first* argument of the function (which must be an object). -The framework will then automatically allow large responses when the user sets -requestMany:true. -*/ - import { isValidUUID } from "@cocalc/util/misc"; import { type Purchases, purchases } from "./purchases"; import { type System, system } from "./system"; import { type Projects, projects } from "./projects"; import { type DB, db } from "./db"; -import { handleErrorMessage } from "@cocalc/nats/util"; +import { type Jupyter, jupyter } from "./jupyter"; +import { handleErrorMessage } from "@cocalc/conat/util"; export interface HubApi { system: System; projects: Projects; db: DB; purchases: Purchases; + jupyter: Jupyter; } const HubApiStructure = { @@ -25,6 +19,7 @@ const HubApiStructure = { projects, db, purchases, + jupyter, } as const; export function transformArgs({ name, args, account_id, project_id }) { @@ -47,7 +42,6 @@ export function initHubApi(callHubApi): HubApi { const resp = await callHubApi({ name: `${group}.${functionName}`, args, - requestMany: args[0]?.requestMany, timeout: args[0]?.timeout, }); return handleErrorMessage(resp); diff --git a/src/packages/conat/hub/api/jupyter.ts b/src/packages/conat/hub/api/jupyter.ts new file mode 100644 index 0000000000..61e4231db5 --- /dev/null +++ b/src/packages/conat/hub/api/jupyter.ts @@ -0,0 +1,23 @@ +import { authFirst } from "./util"; + +export interface Jupyter { + kernels: (opts: { + account_id?: string; + project_id?: string; + }) => Promise; + + execute: (opts: { + input?: string; + kernel?: string; + history?: string[]; + hash?: string; + tag?: string; + project_id?: string; + path?: string; + }) => Promise<{ output: object[]; created: Date } | null>; +} + +export const jupyter = { + kernels: authFirst, + execute: authFirst, +}; diff --git a/src/packages/conat/hub/api/projects.ts b/src/packages/conat/hub/api/projects.ts new file mode 100644 index 0000000000..be1d473d0b --- /dev/null +++ b/src/packages/conat/hub/api/projects.ts @@ -0,0 +1,101 @@ +import { authFirstRequireAccount } from "./util"; +import { type CreateProjectOptions } from "@cocalc/util/db-schema/projects"; +import { type UserCopyOptions } from "@cocalc/util/db-schema/projects"; + +export const projects = { + createProject: authFirstRequireAccount, + copyPathBetweenProjects: authFirstRequireAccount, + removeCollaborator: authFirstRequireAccount, + addCollaborator: authFirstRequireAccount, + inviteCollaborator: authFirstRequireAccount, + inviteCollaboratorWithoutAccount: authFirstRequireAccount, + setQuotas: authFirstRequireAccount, +}; + +export type AddCollaborator = + | { + project_id: string; + account_id: string; + token_id?: undefined; + } + | { + token_id: string; + account_id: string; + project_id?: undefined; + } + | { project_id: string[]; account_id: string[]; token_id?: undefined } // for adding more than one at once + | { account_id: string[]; token_id: string[]; project_id?: undefined }; + +export interface Projects { + // request to have conat permissions to project subjects. + createProject: (opts: CreateProjectOptions) => Promise; + + copyPathBetweenProjects: (opts: UserCopyOptions) => Promise; + + removeCollaborator: ({ + account_id, + opts, + }: { + account_id?: string; + opts: { + account_id; + project_id; + }; + }) => Promise; + + addCollaborator: ({ + account_id, + opts, + }: { + account_id?: string; + opts: AddCollaborator; + }) => Promise<{ project_id?: string | string[] }>; + + inviteCollaborator: ({ + account_id, + opts, + }: { + account_id?: string; + opts: { + project_id: string; + account_id: string; + title?: string; + link2proj?: string; + replyto?: string; + replyto_name?: string; + email?: string; + subject?: string; + }; + }) => Promise; + + inviteCollaboratorWithoutAccount: ({ + account_id, + opts, + }: { + account_id?: string; + opts: { + project_id: string; + title: string; + link2proj: string; + replyto?: string; + replyto_name?: string; + to: string; + email: string; // body in HTML format + subject?: string; + }; + }) => Promise; + + setQuotas: (opts: { + account_id?: string; + project_id: string; + memory?: number; + memory_request?: number; + cpu_shares?: number; + cores?: number; + disk_quota?: number; + mintime?: number; + network?: number; + member_host?: number; + always_running?: number; + }) => Promise; +} diff --git a/src/packages/nats/hub-api/purchases.ts b/src/packages/conat/hub/api/purchases.ts similarity index 100% rename from src/packages/nats/hub-api/purchases.ts rename to src/packages/conat/hub/api/purchases.ts diff --git a/src/packages/nats/hub-api/system.ts b/src/packages/conat/hub/api/system.ts similarity index 59% rename from src/packages/nats/hub-api/system.ts rename to src/packages/conat/hub/api/system.ts index 8f590c16b4..fe3c06196e 100644 --- a/src/packages/nats/hub-api/system.ts +++ b/src/packages/conat/hub/api/system.ts @@ -11,11 +11,16 @@ export const system = { ping: noAuth, terminate: authFirst, userTracking: authFirst, + logClientError: authFirst, + webappError: authFirst, manageApiKeys: authFirst, generateUserAuthToken: authFirst, revokeUserAuthToken: noAuth, userSearch: authFirst, getNames: requireAccount, + adminResetPasswordLink: authFirst, + sendEmailVerification: authFirst, + deletePassport: authFirst, }; export interface System { @@ -34,6 +39,14 @@ export interface System { account_id?: string; }) => Promise; + logClientError: (opts: { + account_id?: string; + event: string; + error: string; + }) => Promise; + + webappError: (opts: object) => Promise; + manageApiKeys: (opts: { account_id?: string; action: ApiKeyAction; @@ -68,4 +81,28 @@ export interface System { } | undefined; }>; + + // adminResetPasswordLink: Enables admins (and only admins!) to generate and get a password reset + // for another user. The response message contains a password reset link, + // though without the site part of the url (the client should fill that in). + // This makes it possible for admins to reset passwords of users, even if + // sending email is not setup, e.g., for cocalc-docker, and also deals with the + // possibility that users have no email address, or broken email, or they + // can't receive email due to crazy spam filtering. + // Non-admins always get back an error. + adminResetPasswordLink: (opts: { + account_id?: string; + user_account_id: string; + }) => Promise; + + sendEmailVerification: (opts: { + account_id?: string; + only_verify?: boolean; + }) => Promise; + + deletePassport: (opts: { + account_id?: string; + strategy: string; + id: string; + }) => Promise; } diff --git a/src/packages/nats/hub-api/util.ts b/src/packages/conat/hub/api/util.ts similarity index 100% rename from src/packages/nats/hub-api/util.ts rename to src/packages/conat/hub/api/util.ts diff --git a/src/packages/conat/hub/changefeeds/client.ts b/src/packages/conat/hub/changefeeds/client.ts new file mode 100644 index 0000000000..c08ffed363 --- /dev/null +++ b/src/packages/conat/hub/changefeeds/client.ts @@ -0,0 +1,66 @@ +import { type Client } from "@cocalc/conat/core/client"; +import { EventIterator } from "@cocalc/util/event-iterator"; +import { getLogger } from "@cocalc/conat/client"; +import { SERVICE, CLIENT_KEEPALIVE, KEEPALIVE_TIMEOUT } from "./util"; +import { ConatError } from "@cocalc/conat/core/client"; + +const logger = getLogger("hub:changefeeds:client"); + +type Update = any; + +function changefeedSubject({ account_id }: { account_id: string }) { + return `${SERVICE}.account-${account_id}`; +} + +export type Changefeed = EventIterator<{ error?: string; update: Update }>; + +export function changefeed({ + query, + options, + client, + account_id, +}: { + query: object; + options?: object[]; + client: Client; + account_id: string; +}) { + const table = Object.keys(query)[0]; + const socket = client.socket.connect(changefeedSubject({ account_id }), { + reconnection: false, + keepAlive: CLIENT_KEEPALIVE, + keepAliveTimeout: KEEPALIVE_TIMEOUT, + desc: `postgresql-changefeed-${table}`, + }); + logger.debug("creating changefeed", { table, options }); + // console.log("creating changefeed", { query, options }); + socket.write({ query, options }); + const cf = new EventIterator<{ error?: string; update: Update }>( + socket, + "data", + { + map: (args) => { + const { error, code, update } = args[0] ?? {}; + if (error) { + // console.log("changefeed: error returned from server, query"); + throw new ConatError(error, { code }); + } else { + return update; + } + }, + onEnd: () => { + // console.log("changefeed: onEnd", query); + socket.close(); + }, + }, + ); + socket.on("closed", () => { + // console.log("changefeed: closed", query); + cf.throw(Error("closed")); + }); + socket.on("disconnected", () => { + // console.log("changefeed: disconnected", query); + cf.throw(Error("disconnected")); + }); + return cf; +} diff --git a/src/packages/conat/hub/changefeeds/index.ts b/src/packages/conat/hub/changefeeds/index.ts new file mode 100644 index 0000000000..2e52a5895a --- /dev/null +++ b/src/packages/conat/hub/changefeeds/index.ts @@ -0,0 +1,2 @@ +export { changefeedServer, type ConatSocketServer } from "./server"; +export { changefeed, type Changefeed } from "./client"; diff --git a/src/packages/conat/hub/changefeeds/server.ts b/src/packages/conat/hub/changefeeds/server.ts new file mode 100644 index 0000000000..1d685c6c82 --- /dev/null +++ b/src/packages/conat/hub/changefeeds/server.ts @@ -0,0 +1,160 @@ +import { type Client, type ConatSocketServer } from "@cocalc/conat/core/client"; +import { uuid } from "@cocalc/util/misc"; +import { UsageMonitor } from "@cocalc/conat/monitor/usage"; +import { getLogger } from "@cocalc/conat/client"; +import { isValidUUID } from "@cocalc/util/misc"; +import { + SUBJECT, + MAX_PER_ACCOUNT, + MAX_GLOBAL, + SERVER_KEEPALIVE, + KEEPALIVE_TIMEOUT, + RESOURCE, +} from "./util"; +export { type ConatSocketServer }; + +const logger = getLogger("hub:changefeeds:server"); + +export function changefeedServer({ + client, + userQuery, + cancelQuery, +}: { + client: Client; + + userQuery: (opts: { + query: object; + options?: object[]; + account_id: string; + changes: string; + cb: Function; + }) => void; + + cancelQuery: (uuid: string) => void; +}): ConatSocketServer { + logger.debug("creating changefeed server"); + + const usage = new UsageMonitor({ + maxPerUser: MAX_PER_ACCOUNT, + max: MAX_GLOBAL, + resource: RESOURCE, + log: (...args) => { + logger.debug(RESOURCE, ...args); + }, + }); + + const server = client.socket.listen(SUBJECT, { + keepAlive: SERVER_KEEPALIVE, + keepAliveTimeout: KEEPALIVE_TIMEOUT, + }); + + server.on("connection", (socket) => { + const v = socket.subject.split(".")[1]; + if (!v?.startsWith("account-")) { + socket.write({ error: "only account users can create changefeeds" }); + logger.debug( + "socket.close: due to changefeed request from non-account subject", + socket.subject, + ); + socket.close(); + return; + } + const account_id = v.slice("account-".length); + if (!isValidUUID(account_id)) { + logger.debug( + "socket.close: due to invalid uuid", + socket.subject, + account_id, + ); + socket.write({ + error: `invalid account_id -- '${account_id}', subject=${socket.subject}`, + }); + socket.close(); + return; + } + let added = false; + try { + usage.add(account_id); + added = true; + } catch (err) { + socket.write({ error: `${err}`, code: err.code }); + logger.debug( + "socket.close: due to usage error (limit exceeded?)", + socket.subject, + err, + ); + socket.close(); + return; + } + + const changes = uuid(); + + socket.on("closed", () => { + logger.debug( + "socket.close: cleaning up since socket closed for some external reason (timeout?)", + socket.subject, + ); + if (added) { + usage.delete(account_id); + } + cancelQuery(changes); + }); + + let running = false; + socket.on("data", (data) => { + if (running) { + socket.write({ error: "exactly one query per connection" }); + logger.debug( + "socket.close: due to attempt to run more than one query", + socket.subject, + ); + socket.close(); + return; + } + running = true; + const { query, options } = data; + try { + userQuery({ + query, + options, + changes, + account_id, + cb: (error, update) => { + // logger.debug("got: ", { error, update }); + try { + socket.write({ error, update }); + } catch (err) { + // happens if buffer is full or socket is closed. in both cases, might was well + // just close the socket. + error = `${err}`; + } + if (error) { + logger.debug( + "socket.close: due to error from postgres changefeed", + socket.subject, + error, + ); + socket.close(); + } + }, + }); + } catch (err) { + logger.debug( + "socket.close: due to error creating query", + socket.subject, + err, + ); + try { + socket.write({ error: `${err}` }); + } catch {} + socket.close(); + } + }); + }); + server.on("closed", () => { + logger.debug("shutting down changefeed server"); + usage.close(); + }); + + return server; +} diff --git a/src/packages/conat/hub/changefeeds/util.ts b/src/packages/conat/hub/changefeeds/util.ts new file mode 100644 index 0000000000..881dc6015f --- /dev/null +++ b/src/packages/conat/hub/changefeeds/util.ts @@ -0,0 +1,27 @@ +export const SERVICE = "changefeeds"; +export const SUBJECT = "changefeeds.*"; + +// This is the max *per account* connected to a single server, just +// because everything should have limits. +// If the user refreshes their browser, it is still about a minute +// before all the changefeeds they had open are free (due to the +// SERVER_KEEPALIVE time below). +export const MAX_PER_ACCOUNT = 500; +export const MAX_GLOBAL = 10000; + +const DEBUG_DEVEL_MODE = false; + +export let CLIENT_KEEPALIVE = 90000; +export let SERVER_KEEPALIVE = 45000; +export let KEEPALIVE_TIMEOUT = 10000; + +if (DEBUG_DEVEL_MODE) { + console.log( + "*** WARNING: Using DEBUB_DEVEL_MODE changefeed parameters!! ***", + ); + CLIENT_KEEPALIVE = 6000; + SERVER_KEEPALIVE = 3000; + KEEPALIVE_TIMEOUT = 1000; +} + +export const RESOURCE = "PostgreSQL changefeeds"; diff --git a/src/packages/conat/jest.config.js b/src/packages/conat/jest.config.js new file mode 100644 index 0000000000..bd29bb9761 --- /dev/null +++ b/src/packages/conat/jest.config.js @@ -0,0 +1,6 @@ +/** @type {import('ts-jest').JestConfigWithTsJest} */ +module.exports = { + preset: "ts-jest", + testEnvironment: "node", + testMatch: ["**/?(*.)+(spec|test).ts?(x)"], +}; diff --git a/src/packages/nats/llm/client.ts b/src/packages/conat/llm/client.ts similarity index 73% rename from src/packages/nats/llm/client.ts rename to src/packages/conat/llm/client.ts index 4613b18d8f..4745c06a30 100644 --- a/src/packages/nats/llm/client.ts +++ b/src/packages/conat/llm/client.ts @@ -1,12 +1,11 @@ /* -Client for the nats server in server.ts. +Client for the conat server in server.ts. */ -import { getEnv } from "@cocalc/nats/client"; +import { conat } from "@cocalc/conat/client"; import type { ChatOptions } from "@cocalc/util/types/llm"; import { isValidUUID } from "@cocalc/util/misc"; import { llmSubject } from "./server"; -import { waitUntilConnected } from "@cocalc/nats/util"; export async function llm(options: ChatOptions): Promise { if (!options.system?.trim()) { @@ -20,18 +19,17 @@ export async function llm(options: ChatOptions): Promise { let all = ""; let lastSeq = -1; - const { nc, jc } = await getEnv(); + const cn = await conat(); let { stream, ...opts } = options; - await waitUntilConnected(); - for await (const resp of await nc.requestMany(subject, jc.encode(opts), { + for await (const resp of await cn.requestMany(subject, opts, { maxWait: opts.timeout ?? 1000 * 60 * 10, })) { - if (resp.data.length == 0) { + if (resp.data == null) { // client code also expects null token to know when stream is done. stream?.(null); break; } - const { error, text, seq } = jc.decode(resp.data); + const { error, text, seq } = resp.data; if (error) { throw Error(error); } diff --git a/src/packages/nats/llm/server.ts b/src/packages/conat/llm/server.ts similarity index 85% rename from src/packages/nats/llm/server.ts rename to src/packages/conat/llm/server.ts index 58fbcbe519..77afd7d86d 100644 --- a/src/packages/nats/llm/server.ts +++ b/src/packages/conat/llm/server.ts @@ -1,5 +1,5 @@ /* -Multiresponse request/response NATS server that makes +Multiresponse request/response conat server that makes llm's available to users of CoCalc. Query with a multiResponse request at this subject @@ -11,9 +11,9 @@ it so projects can directly use llm's... but first we need to figure out how paying for that would work. */ -import { getEnv } from "@cocalc/nats/client"; -import { type Subscription, Empty } from "@nats-io/nats-core"; +import { conat } from "@cocalc/conat/client"; import { isValidUUID } from "@cocalc/util/misc"; +import type { Subscription } from "@cocalc/conat/core/client"; export const SUBJECT = process.env.COCALC_TEST_MODE ? "llm-test" : "llm"; @@ -52,8 +52,8 @@ function getUserId(subject: string): string { let sub: Subscription | null = null; export async function init(evaluate) { - const { nc } = await getEnv(); - sub = nc.subscribe(`${SUBJECT}.*.api`, { queue: "q" }); + const cn = await conat(); + sub = await cn.subscribe(`${SUBJECT}.*.api`, { queue: "q" }); listen(evaluate); } @@ -75,12 +75,11 @@ async function listen(evaluate) { } async function handleMessage(mesg, evaluate) { - const { jc } = await getEnv(); - const options = jc.decode(mesg.data); + const options = mesg.data; let seq = 0; const respond = ({ text, error }: { text?: string; error?: string }) => { - mesg.respond(jc.encode({ text, error, seq })); + mesg.respondSync({ text, error, seq }); seq += 1; }; @@ -88,8 +87,8 @@ async function handleMessage(mesg, evaluate) { const end = () => { if (done) return; done = true; - // end response stream with empty payload. - mesg.respond(Empty); + // end response stream with null payload. + mesg.respondSync(null); }; const stream = (text?) => { diff --git a/src/packages/conat/monitor/tables.ts b/src/packages/conat/monitor/tables.ts new file mode 100644 index 0000000000..d96b2b5b44 --- /dev/null +++ b/src/packages/conat/monitor/tables.ts @@ -0,0 +1,168 @@ +/* +Displaying ASCII art tables in the terminal to understand Conat state. + +We will also have similar functionality in the web app. Both are a good idea to +have for various reasons. + + +*/ + +import { AsciiTable3 } from "ascii-table3"; +import { type Client } from "@cocalc/conat/core/client"; +import { field_cmp, human_readable_size } from "@cocalc/util/misc"; +import dayjs from "dayjs"; +import duration from "dayjs/plugin/duration"; + +dayjs.extend(duration); + +function formatCompactDuration(ms: number): string { + const d = dayjs.duration(ms); + + const hours = d.hours(); + const minutes = d.minutes(); + const seconds = d.seconds(); + + let out = ""; + if (d.asDays() >= 1) out += `${Math.floor(d.asDays())}d`; + if (d.asHours() % 24 >= 1) out += `${hours}h`; + if (d.asMinutes() % 60 >= 1) out += `${minutes}m`; + out += `${seconds}s`; + return out; +} + +interface Options { + client: Client; + maxWait?: number; + maxMessages?: number; +} + +// cd packages/backend; pnpm conat-connections +export async function usage({ client, maxWait = 3000, maxMessages }: Options) { + const sys = client.callMany("sys.conat.server", { maxWait, maxMessages }); + const data = await sys.usage(); + const rows: any[] = []; + const perServerRows: any[] = []; + let total = 0; + for await (const X of data) { + for (const server in X) { + const { perUser, total: total0 } = X[server]; + perServerRows.push([server, total0]); + total += total0; + + for (const user in perUser) { + rows.push([server, user, perUser[user]]); + } + } + } + rows.sort(field_cmp("2")); + rows.push(["", "", ""]); + rows.push(["TOTAL", "", total]); + + const tablePerUser = new AsciiTable3(`${total} Connections`) + .setHeading("Server", "User", "Connections") + .addRowMatrix(rows); + const tablePerServer = new AsciiTable3(`Connections Per Server`) + .setHeading("Server", "Connections") + .addRowMatrix(perServerRows); + + tablePerUser.setStyle("unicode-round"); + tablePerServer.setStyle("unicode-round"); + + return [tablePerUser, tablePerServer]; +} + +export async function stats({ client, maxWait = 3000, maxMessages }: Options) { + const sys = client.callMany("sys.conat.server", { maxWait, maxMessages }); + const data = await sys.stats(); + + const rows: any[] = []; + const cols = 10; + const totals = Array(cols).fill(0); + for await (const X of data) { + for (const server in X) { + const stats = X[server]; + for (const id in stats) { + const x = stats[id]; + let user; + if (x.user?.error) { + user = x.user.error; + } else { + user = JSON.stringify(x.user).slice(1, -1); + } + const uptime = formatCompactDuration(Date.now() - x.connected); + rows.push([ + id, + user, + server, + x.address, + uptime, + x.recv.messages, + x.send.messages, + human_readable_size(x.recv.bytes), + human_readable_size(x.send.bytes), + x.subs, + ]); + totals[cols - 5] += x.recv.messages; + totals[cols - 4] += x.send.messages; + totals[cols - 3] += x.recv.bytes; + totals[cols - 2] += x.send.bytes; + totals[cols - 1] += x.subs; + } + } + } + rows.sort(field_cmp(`${cols - 1}`)); + rows.push(Array(cols).fill("")); + rows.push([ + "TOTALS", + `Total for ${rows.length - 1} connections:`, + ...Array(cols - 7).fill(""), + totals[cols - 5], + totals[cols - 4], + human_readable_size(totals[cols - 3]), + human_readable_size(totals[cols - 2]), + totals[cols - 1], + ]); + + const table = new AsciiTable3(`${rows.length - 2} Conat Connections`) + .setHeading( + "ID", + "User", + "Server", + "Address", + "Uptime", + "In Msgs", + "Out Msgs", + "In Bytes", + "Out Bytes", + "Subs", + ) + .addRowMatrix(rows); + + table.setStyle("unicode-round"); + return [table]; +} + +export async function showUsersAndStats({ + client, + maxWait = 3000, + maxMessages, +}: Options) { + let s; + if (maxMessages) { + s = `for up ${maxMessages} servers `; + } else { + s = ""; + } + console.log(`Gather data ${s}for up to ${maxWait / 1000} seconds...\n\n`); + const X = [stats, usage]; + const tables: any[] = []; + const f = async (i) => { + for (const table of await X[i]({ client, maxWait, maxMessages })) { + tables.push(table); + } + }; + await Promise.all([f(0), f(1)]); + for (const table of tables) { + console.log(table.toString()); + } +} diff --git a/src/packages/conat/monitor/usage.ts b/src/packages/conat/monitor/usage.ts new file mode 100644 index 0000000000..9e09f2444e --- /dev/null +++ b/src/packages/conat/monitor/usage.ts @@ -0,0 +1,100 @@ +import json from "json-stable-stringify"; +import { EventEmitter } from "events"; +import type { JSONValue } from "@cocalc/util/types"; +import { ConatError } from "@cocalc/conat/core/client"; +import { getLogger } from "@cocalc/conat/client"; + +const logger = getLogger("monitor:usage"); + +interface Options { + resource: string; + maxPerUser?: number; + max?: number; + log?: (...args) => void; +} + +export class UsageMonitor extends EventEmitter { + private options: Options; + private total = 0; + private perUser: { [user: string]: number } = {}; + + constructor(options: Options) { + super(); + this.options = options; + logger.debug("creating usage monitor", this.options); + this.initLogging(); + } + + stats = () => { + return { total: this.total, perUser: this.perUser }; + }; + + close = () => { + this.removeAllListeners(); + this.perUser = {}; + }; + + private toJson = (user: JSONValue) => json(user) ?? ""; + + private initLogging = () => { + const { log } = this.options; + if (log == null) { + return; + } + this.on("total", (total, limit) => { + log("usage", this.options.resource, { total, limit }); + }); + this.on("add", (user, count, limit) => { + log("usage", this.options.resource, "add", { user, count, limit }); + }); + this.on("delete", (user, count, limit) => { + log("usage", this.options.resource, "delete", { user, count, limit }); + }); + this.on("deny", (user, limit, type) => { + log("usage", this.options.resource, "not allowed due to hitting limit", { + type, + user, + limit, + }); + }); + }; + + add = (user: JSONValue) => { + const u = this.toJson(user); + let count = this.perUser[u] ?? 0; + if (this.options.max && this.total >= this.options.max) { + this.emit("deny", user, this.options.max, "global"); + throw new ConatError( + `There is a global limit of ${this.options.max} ${this.options.resource}. Please close browser tabs or files or come back later.`, + // http error code "429 Too Many Requests." + { code: 429 }, + ); + } + if (this.options.maxPerUser && count >= this.options.maxPerUser) { + this.emit("deny", this.options.maxPerUser, "per-user"); + throw new ConatError( + `There is a per user limit of ${this.options.maxPerUser} ${this.options.resource}. Please close browser tabs or files or come back later.`, + // http error code "429 Too Many Requests." + { code: 429 }, + ); + } + this.total += 1; + count++; + this.perUser[u] = count; + this.emit("total", this.total, this.options.max); + this.emit("add", user, count, this.options.maxPerUser); + }; + + delete = (user: JSONValue) => { + this.total -= 1; + const u = this.toJson(user); + let count = (this.perUser[u] ?? 0) - 1; + if (count <= 0) { + delete this.perUser[u]; + } else { + this.perUser[u] = count; + } + this.emit("total", this.total); + this.emit("delete", user, count); + }; +} diff --git a/src/packages/nats/names.ts b/src/packages/conat/names.ts similarity index 78% rename from src/packages/nats/names.ts rename to src/packages/conat/names.ts index b01d071dfc..3d6b1dab5c 100644 --- a/src/packages/nats/names.ts +++ b/src/packages/conat/names.ts @@ -1,5 +1,5 @@ /* -Names we use with nats. +Names we use with conat. For Jetstream: @@ -13,9 +13,9 @@ For Subjects: import generateVouchers from "@cocalc/util/vouchers"; import type { Location } from "./types"; -import { encodeBase64 } from "@cocalc/nats/util"; +import { encodeBase64 } from "@cocalc/conat/util"; -// nice alphanumeric string that can be used as nats subject, and very +// nice alphanumeric string that can be used as conat subject, and very // unlikely to randomly collide with another browser tab from this account. export function randomId() { return generateVouchers({ count: 1, length: 10 })[0]; @@ -26,24 +26,26 @@ export function randomId() { export function jsName({ project_id, account_id, + hub_id, }: { project_id?: string; account_id?: string; + hub_id?: string; }) { if (project_id) { - if (account_id) { - throw Error("both account_id and project_id can't be set"); - } return `project-${project_id}`; } - if (!account_id) { - if (process.env.COCALC_TEST_MODE) { - return "test"; - } else { - return "public"; - } + if (account_id) { + return `account-${account_id}`; + } + if (hub_id) { + return `hub-${hub_id}`; + } + if (process.env.COCALC_TEST_MODE) { + return "test"; + } else { + return "public"; } - return `account-${account_id}`; } export function localLocationName({ @@ -72,30 +74,19 @@ Custom inbox prefix per "user"! So can receive response to requests, and that you can ONLY receive responses to your own messages and nobody else's! This must be used in conjunction with -the inboxPrefix client option when connecting. Note that the NATS docs - https://docs.nats.io/running-a-nats-service/configuration/securing_nats/authorization -do not explain this, instead just emphasizing you're screwed but not giving -the solution, which is very disconcerting! There are a couple of places in -our code where we create connections, and these all must be aware of the -inbox prefix we use. - -This is explained in this natsbyexample page: - -https://natsbyexample.com/examples/auth/private-inbox/cli +the inboxPrefix client option when connecting. */ export function inboxPrefix({ account_id, project_id, + hub_id, }: { account_id?: string; project_id?: string; + hub_id?: string; }) { - if (!account_id && !project_id) { - // the hubs - return "_INBOX.hub"; - } // a project or account: - return `_INBOX.${jsName({ account_id, project_id })}`; + return `_INBOX.${jsName({ account_id, project_id, hub_id })}`; } export function streamSubject({ diff --git a/src/packages/nats/package.json b/src/packages/conat/package.json similarity index 54% rename from src/packages/nats/package.json rename to src/packages/conat/package.json index acc34d9d60..4338536ec4 100644 --- a/src/packages/nats/package.json +++ b/src/packages/conat/package.json @@ -1,24 +1,28 @@ { - "name": "@cocalc/nats", + "name": "@cocalc/conat", "version": "1.0.0", - "description": "CoCalc NATS integration code. Usable by both nodejs and browser.", + "description": "Conat -- pub/sub framework. Usable by both nodejs and browser.", "exports": { "./sync/*": "./dist/sync/*.js", "./llm/*": "./dist/llm/*.js", - "./hub-api": "./dist/hub-api/index.js", - "./hub-api/*": "./dist/hub-api/*.js", + "./socket": "./dist/socket/index.js", + "./socket/*": "./dist/socket/*.js", + "./hub/changefeeds": "./dist/hub/changefeeds/index.js", + "./hub/api": "./dist/hub/api/index.js", + "./hub/api/*": "./dist/hub/api/*.js", "./compute/*": "./dist/compute/*.js", "./service": "./dist/service/index.js", - "./project-api": "./dist/project-api/index.js", + "./project/api": "./dist/project/api/index.js", "./browser-api": "./dist/browser-api/index.js", "./*": "./dist/*.js" }, "scripts": { "preinstall": "npx only-allow pnpm", "build": "pnpm exec tsc --build", + "clean": "rm -rf dist node_modules", "tsc": "pnpm exec tsc --watch --pretty --preserveWatchOutput", - "prepublishOnly": "pnpm test", - "test": "echo 'see packages/backend/nats/tests instead'" + "test": "pnpm exec jest", + "depcheck": "pnpx depcheck --ignores events" }, "files": [ "dist/**", @@ -28,26 +32,33 @@ "author": "SageMath, Inc.", "keywords": [ "utilities", - "nats", + "conat", "cocalc" ], "license": "SEE LICENSE.md", "dependencies": { "@cocalc/comm": "workspace:*", - "@cocalc/nats": "workspace:*", + "@cocalc/conat": "workspace:*", + "@cocalc/redis-streams-adapter": "^0.2.3", "@cocalc/util": "workspace:*", - "@nats-io/jetstream": "3.0.0", - "@nats-io/kv": "3.0.0", - "@nats-io/nats-core": "3.0.0", - "@nats-io/services": "3.0.0", + "@isaacs/ttlcache": "^1.4.1", + "@msgpack/msgpack": "^3.1.1", + "@socket.io/redis-adapter": "^8.3.0", + "ascii-table3": "^1.0.1", "awaiting": "^3.0.0", + "consistent-hash": "^1.2.2", + "dayjs": "^1.11.11", "events": "3.3.0", "immutable": "^4.3.0", + "iovalkey": "^0.3.1", "js-base64": "^3.7.7", "json-stable-stringify": "^1.0.1", - "lodash": "^4.17.21" + "lodash": "^4.17.21", + "socket.io": "^4.8.1", + "socket.io-client": "^4.8.1" }, "devDependencies": { + "@types/better-sqlite3": "^7.6.13", "@types/json-stable-stringify": "^1.0.32", "@types/lodash": "^4.14.202", "@types/node": "^18.16.14" @@ -56,5 +67,5 @@ "type": "git", "url": "https://github.com/sagemathinc/cocalc" }, - "homepage": "https://github.com/sagemathinc/cocalc/tree/master/src/packages/nats" + "homepage": "https://github.com/sagemathinc/cocalc/tree/master/src/packages/conat" } diff --git a/src/packages/conat/persist/README.md b/src/packages/conat/persist/README.md new file mode 100644 index 0000000000..a40ebf8797 --- /dev/null +++ b/src/packages/conat/persist/README.md @@ -0,0 +1,93 @@ +# Persistence Service + +The goal is to provide a tiered, memory efficient, scalable way to persist +streams and kv stores, without using Jetstream. This should use only the core +pub/sub functionality of NATS, so we can switch to other pub/sub systems later. + +## API + +Given a subject that the requesting user has access to, this service can do the following. + +Message = {value?:Buffer, headers?:Headers, timestamp?:number} + +- set: \(subject, seq:number, message}\) +- get: \({subject, seq:number}\) => Message +- delete: \({subject, seq:number}\) +- getAll: \({subject, start\_seq?:number}\) => Message\[\], as sequence of messages + - if start\_seq given, gets only messages >= start\_seq +- deleteAll:\({subject, end\_seq?:number}\) + - if end\_seq given, deletes only messages <= end\_seq + +Moreover, every time one client makes a change, a corresponding message gets +published so all other clients can update their state. This will use exactly +the protocol implemented in core-stream.ts right now. + +Notes: + +- We use chunking so there are no limits on message size. +- There is no history for kv, i.e., only the last value is saved. (kv is **not** implemented + on top of streams like in NATS; it is its own thing) +- Messages can be deleted in a stream. + +## Architecture: + +- There can be a large number of persist services. These servers a single threaded and + may require substantial RAM and cpu to do their work, so we have to be able easily scale + the number up and down. + +- Each stream storage server has: + + - mounts a common shared filesystem across all persistence servers. + - (optional) access to a common cloud storage bucket for longterm cheap + high-latency tiered storage. + +- One load balancer that decided which persistence server should server a given stream. + + - coordinator persists its state to the common shared filesystem as well. + - This defines a map + + (stream) |--> (persist server) + + that changes only when a persist server terminates. + The persist servers send periodic heartbeats to coordinator and the coordinator + allocates stream work ONLY to persist servers that have sent a recent heartbeat. + - When coordinator is restarted there's a short period when new clients can't + open a stream. Existing clients keep using the streams as before. + - The obvious problem with this approach is if persist server A is working fine + but somehow communication with the coordinator stops, then the coordinator + switches the stream to use persist server B and some clients use B, but some + clients are still using persist server A. Basically, split brain. + If this happened though server A and server B are still using the same sqlite + file (over NFS) so there's still locking at the NFS level. The loss would be + that users would not see each other's changes. If there's split brain though, + that means our pub/sub layer is fundamentally broken, so it's acceptable that + users aren't seeing each other's changes in such a scenario. + + +Requirements: + + - must scale up a lot, e..g, imagine 10,000 simultaneous users, doing a lot with terminals, editing, jupyter, etc., all at once -- that's well over 10K+ messages/second to this system + - efficient in terms of cost + - a minute of downtime for a subset of streams once in a while is ok; global downtime for all streams would be very bad. + - very small amount of data loss (e.g., last few seconds of edit history) is ok + + + +## Protocol: + +- When any client wants to use a subject, it makes a request to the coordinator asking which + persistence server it should use. The coordinator selects from active persistence servers + and it makes a consistent assignment. If a persistence servers stops working or vanishes, + clients will again make a request, and the coordinator will answer, possibly with a + different server. + - A persistence server is the analogue of a NATS jetstream node. We use + a coordinator so there is no need for RAFT. Using cloud storage provides + tiered storage. Only accessing the sqlite file when there's a request lets + us scale to an unlimited number of subjects but maintain very fast + startup time. +- Client makes requests as mentioned above to a specific named persistence server. + +- When server gets such a request, it opens the subject by copying the sqlite3 file from + cloud storage to a local disk if necessary, then queries it and responds. +- Periodically the server copies the sqlite3 file from local disk to cloud storage. + diff --git a/src/packages/conat/persist/auth.ts b/src/packages/conat/persist/auth.ts new file mode 100644 index 0000000000..5baa1f37f4 --- /dev/null +++ b/src/packages/conat/persist/auth.ts @@ -0,0 +1,78 @@ +import { SERVICE } from "./util"; +import { ConatError } from "@cocalc/conat/core/client"; + +export const MAX_PATH_LENGTH = 4000; + +export function getUserId(subject: string): string { + if ( + subject.startsWith(`${SERVICE}.account-`) || + subject.startsWith(`${SERVICE}.project-`) + ) { + // note that project and account have the same number of letters + return subject.slice( + `${SERVICE}.account-`.length, + `${SERVICE}.account-`.length + 36, + ); + } + return ""; +} + +export function assertHasWritePermission({ + subject, + path, +}: { + // Subject definitely has one of the following forms, or we would never + // see this message: + // ${SERVICE}.account-${account_id}.> or + // ${SERVICE}.project-${project_id}.> or + // ${SERVICE}.hub.> + // ${SERVICE}.SOMETHING-WRONG + // A user is only allowed to write to a subject if they have rights + // to the given project, account or are a hub. + // The path can a priori be any string. However, here's what's allowed + // accounts/[account_id]/any...thing + // projects/[project_id]/any...thing + // hub/any...thing <- only hub can write to this. + subject: string; + path: string; +}) { + if (path.length > MAX_PATH_LENGTH) { + throw new ConatError( + `permission denied: path (of length ${path.length}) is too long (limit is '${MAX_PATH_LENGTH}' characters)`, + { code: 403 }, + ); + } + if (path.includes("..") || path.startsWith("/") || path.endsWith("/")) { + throw new ConatError( + `permission denied: path '${path}' must not include .. or start or end with / `, + { code: 403 }, + ); + } + const v = subject.split("."); + if (v[0] != SERVICE) { + throw Error( + `bug -- first segment of subject must be ${SERVICE} -- subject=${subject}`, + ); + } + const s = v[1]; + if (s == "hub") { + // hub user can write to any path + return; + } + for (const cls of ["account", "project"]) { + if (s.startsWith(cls + "-")) { + const user_id = getUserId(subject); + const base = cls + "s/" + user_id + "/"; + if (path.startsWith(base)) { + // permissions granted + return; + } else { + throw new ConatError( + `permission denied: subject '${subject}' does not grant write permission to path='${path}' since it is not under '${base}'`, + { code: 403 }, + ); + } + } + } + throw new ConatError(`invalid subject: '${subject}'`, { code: 403 }); +} diff --git a/src/packages/conat/persist/client.ts b/src/packages/conat/persist/client.ts new file mode 100644 index 0000000000..80c01d3b24 --- /dev/null +++ b/src/packages/conat/persist/client.ts @@ -0,0 +1,393 @@ +import { + type Message as ConatMessage, + type Client, + type MessageData, + ConatError, +} from "@cocalc/conat/core/client"; +import { type ConatSocketClient } from "@cocalc/conat/socket"; +import { EventIterator } from "@cocalc/util/event-iterator"; +import type { + StorageOptions, + Configuration, + SetOperation, + DeleteOperation, + StoredMessage, + PartialInventory, +} from "./storage"; +export { StoredMessage, StorageOptions }; +import { persistSubject, type User } from "./util"; +import { assertHasWritePermission as assertHasWritePermission0 } from "./auth"; +import { refCacheSync } from "@cocalc/util/refcache"; +import { EventEmitter } from "events"; +import { getLogger } from "@cocalc/conat/client"; + +const logger = getLogger("persist:client"); + +export interface ChangefeedEvent { + updates: (SetOperation | DeleteOperation)[]; + seq: number; +} + +export type Changefeed = EventIterator; + +// const paths = new Set(); + +export { type PersistStreamClient }; +class PersistStreamClient extends EventEmitter { + public socket: ConatSocketClient; + private changefeeds: any[] = []; + private state: "ready" | "closed" = "ready"; + + constructor( + private client: Client, + private storage: StorageOptions, + private user: User, + ) { + super(); + // paths.add(this.storage.path); + logger.debug("constructor", this.storage); + this.init(); + } + + private init = () => { + if (this.client.state == "closed") { + this.close(); + return; + } + if (this.state == "closed") { + return; + } + this.socket?.close(); + // console.log("making a socket connection to ", persistSubject(this.user)); + this.socket = this.client.socket.connect(persistSubject(this.user), { + desc: `persist: ${this.storage.path}`, + reconnection: false, + }); + logger.debug( + "init", + this.storage.path, + "connecting to ", + persistSubject(this.user), + ); + // console.log( + // "persist -- create", + // this.storage.path, + // paths, + // "with id=", + // this.socket.id, + // ); + this.socket.write({ + storage: this.storage, + changefeed: this.changefeeds.length > 0, + }); + + this.socket.once("disconnected", () => { + this.socket.removeAllListeners(); + setTimeout(this.init, 1000); + }); + this.socket.once("closed", () => { + this.socket.removeAllListeners(); + setTimeout(this.init, 1000); + }); + + this.socket.on("data", (updates, headers) => { + if (updates == null && headers != null) { + // has to be an error + this.emit( + "error", + new ConatError(headers?.error, { code: headers?.code }), + ); + this.close(); + } + this.emit("changefeed", { updates, seq: headers?.seq }); + }); + }; + + close = () => { + logger.debug("close", this.storage); + // paths.delete(this.storage.path); + // console.log("persist -- close", this.storage.path, paths); + this.state = "closed"; + this.emit("closed"); + for (const iter of this.changefeeds) { + iter.close(); + this.changefeeds.length = 0; + } + this.socket.close(); + }; + + changefeed = async (): Promise => { + // activate changefeed mode (so server publishes updates -- this is idempotent) + const resp = await this.socket.request(null, { + headers: { + cmd: "changefeed", + }, + }); + if (resp.headers?.error) { + throw new ConatError(`${resp.headers?.error}`, { + code: resp.headers?.code, + }); + } + // an iterator over any updates that are published. + const iter = new EventIterator(this, "changefeed", { + map: (args) => args[0], + }); + this.changefeeds.push(iter); + return iter; + }; + + set = async ({ + key, + ttl, + previousSeq, + msgID, + messageData, + timeout, + }: SetOptions & { timeout?: number }): Promise<{ + seq: number; + time: number; + }> => { + return this.checkForError( + await this.socket.request(null, { + raw: messageData.raw, + encoding: messageData.encoding, + headers: { + headers: messageData.headers, + cmd: "set", + key, + ttl, + previousSeq, + msgID, + timeout, + }, + timeout, + }), + ); + }; + + setMany = async ( + ops: SetOptions[], + { timeout }: { timeout?: number } = {}, + ): Promise< + ({ seq: number; time: number } | { error: string; code?: any })[] + > => { + return this.checkForError( + await this.socket.request(ops, { + headers: { + cmd: "setMany", + timeout, + }, + timeout, + }), + ); + }; + + delete = async ({ + timeout, + seq, + last_seq, + all, + }: { + timeout?: number; + seq?: number; + last_seq?: number; + all?: boolean; + }): Promise<{ seqs: number[] }> => { + return this.checkForError( + await this.socket.request(null, { + headers: { + cmd: "delete", + seq, + last_seq, + all, + timeout, + }, + timeout, + }), + ); + }; + + config = async ({ + config, + timeout, + }: { + config?: Partial; + timeout?: number; + } = {}): Promise => { + return this.checkForError( + await this.socket.request(null, { + headers: { + cmd: "config", + config, + timeout, + } as any, + timeout, + }), + ); + }; + + inventory = async (timeout?): Promise => { + return this.checkForError( + await this.socket.request(null, { + headers: { + cmd: "inventory", + } as any, + timeout, + }), + ); + }; + + get = async ({ + seq, + key, + timeout, + }: { + timeout?: number; + } & ( + | { seq: number; key?: undefined } + | { key: string; seq?: undefined } + )): Promise => { + const resp = await this.socket.request(null, { + headers: { cmd: "get", seq, key, timeout } as any, + timeout, + }); + this.checkForError(resp, true); + if (resp.headers == null) { + return undefined; + } + return resp; + }; + + // returns async iterator over arrays of stored messages + async *getAll({ + start_seq, + end_seq, + timeout, + maxWait, + }: { + start_seq?: number; + end_seq?: number; + timeout?: number; + maxWait?: number; + } = {}): AsyncGenerator { + const sub = await this.socket.requestMany(null, { + headers: { + cmd: "getAll", + start_seq, + end_seq, + timeout, + } as any, + timeout, + maxWait, + }); + for await (const { data, headers } of sub) { + if (headers?.error) { + throw new ConatError(`${headers.error}`, { code: headers.code }); + } + if (data == null || this.socket.state == "closed") { + // done + return; + } + yield data; + } + } + + keys = async ({ timeout }: { timeout?: number } = {}): Promise => { + return this.checkForError( + await this.socket.request(null, { + headers: { cmd: "keys", timeout } as any, + timeout, + }), + ); + }; + + sqlite = async ({ + timeout, + statement, + params, + }: { + timeout?: number; + statement: string; + params?: any[]; + }): Promise => { + return this.checkForError( + await this.socket.request(null, { + headers: { + cmd: "sqlite", + statement, + params, + } as any, + timeout, + }), + ); + }; + + private checkForError = (mesg, noReturn = false) => { + if (mesg.headers != null) { + const { error, code } = mesg.headers; + if (error || code) { + throw new ConatError(error ?? "error", { code }); + } + } + if (!noReturn) { + return mesg.data; + } + }; + + // id of the remote server we're connected to + serverId = async () => { + return this.checkForError( + await this.socket.request(null, { + headers: { cmd: "serverId" }, + }), + ); + }; +} + +export interface SetOptions { + messageData: MessageData; + key?: string; + ttl?: number; + previousSeq?: number; + msgID?: string; + timeout?: number; +} + +interface Options { + client: Client; + // who is accessing persistent storage + user: User; + // what storage they are accessing + storage: StorageOptions; + noCache?: boolean; +} + +export const stream = refCacheSync({ + name: "persistent-stream-client", + createKey: ({ user, storage, client }: Options) => { + return JSON.stringify([user, storage, client.id]); + }, + createObject: ({ client, user, storage }: Options) => { + // avoid wasting server resources, etc., by always checking permissions client side first + assertHasWritePermission({ user, storage }); + return new PersistStreamClient(client, storage, user); + }, +}); + +let permissionChecks = true; +export function disablePermissionCheck() { + if (!process.env.COCALC_TEST_MODE) { + throw Error("disabling permission check only allowed in test mode"); + } + permissionChecks = false; +} + +const assertHasWritePermission = ({ user, storage }) => { + if (!permissionChecks) { + // should only be used for unit testing, since otherwise would + // make clients slower and possibly increase server load. + return; + } + const subject = persistSubject(user); + assertHasWritePermission0({ subject, path: storage.path }); +}; diff --git a/src/packages/conat/persist/context.ts b/src/packages/conat/persist/context.ts new file mode 100644 index 0000000000..67beeedbfb --- /dev/null +++ b/src/packages/conat/persist/context.ts @@ -0,0 +1,51 @@ +/* +Define functions for using sqlite, the filesystem, compression, etc. +These are functions that typically get set via nodejs on the backend, +not from a browser. Making this explicit helps clarify the dependence +on the backend and make the code more unit testable. +*/ + +import type BetterSqlite3 from "better-sqlite3"; +type Database = BetterSqlite3.Database; +export { type Database }; + +let betterSqlite3: any = null; + +export let compress: (data: Buffer) => Buffer = () => { + throw Error("must initialize persist context"); +}; + +export let decompress: (data: Buffer) => Buffer = () => { + throw Error("must initialize persist context"); +}; + +export let syncFiles = { local: "", archive: "" }; + +export let ensureContainingDirectoryExists: (path: string) => Promise = ( + _path, +) => { + throw Error("must initialize persist context"); +}; + +export function initContext(opts: { + betterSqlite3; + compress: (Buffer) => Buffer; + decompress: (Buffer) => Buffer; + syncFiles: { local: string; archive: string }; + ensureContainingDirectoryExists: (path: string) => Promise; +}) { + betterSqlite3 = opts.betterSqlite3; + compress = opts.compress; + decompress = opts.decompress; + syncFiles = opts.syncFiles; + ensureContainingDirectoryExists = opts.ensureContainingDirectoryExists; +} + +export function createDatabase(...args): Database { + if (betterSqlite3 == null) { + throw Error( + "conat/persist must be initialized with the better-sqlite3 module -- import from backend/conat/persist instead", + ); + } + return new betterSqlite3(...args); +} diff --git a/src/packages/conat/persist/server.ts b/src/packages/conat/persist/server.ts new file mode 100644 index 0000000000..78ba5a33ad --- /dev/null +++ b/src/packages/conat/persist/server.ts @@ -0,0 +1,359 @@ +/* +CONAT_SERVER=http://localhost:3000 node + +// making a server from scratch + +// initialize persist context + +require('@cocalc/backend/conat/persist'); + +// a conat server and client +s = require('@cocalc/conat/core/server').init({port:4567, getUser:()=>{return {hub_id:'hub'}}}); client = s.client(); + +// persist server +p = require('@cocalc/conat/persist/server').server({client}); 0; + + + +// a client for persist server + +c = require('@cocalc/conat/persist/client').stream({client, user:{hub_id:'hub'}, storage:{path:'b.txt'}}); + +for await (x of await c.getAll()) { console.log(x) } + + +await c.set({messageData:client.message(123)}) + +for await (x of await c.getAll()) { console.log(x) } + +[ { seq: 1, time: 1750218209211, encoding: 0, raw: } ] + +(await c.get({seq:5})).data + +await c.set({key:'foo', messageData:client.message('bar')}) +(await c.get({key:'foo'})).data + +await c.delete({seq:6}) + + +client = await require('@cocalc/backend/conat').conat(); kv = require('@cocalc/backend/conat/sync').akv({project_id:'3fa218e5-7196-4020-8b30-e2127847cc4f', name:'a.txt', client}) + +client = await require('@cocalc/backend/conat').conat(); s = require('@cocalc/backend/conat/sync').astream({project_id:'3fa218e5-7196-4020-8b30-e2127847cc4f', name:'b.txt', client}) + +client = await require('@cocalc/backend/conat').conat(); s = await require('@cocalc/backend/conat/sync').dstream({project_id:'3fa218e5-7196-4020-8b30-e2127847cc4f', name:'ds2.txt', client}) + + +client = await require('@cocalc/backend/conat').conat(); kv = require('@cocalc/backend/conat/sync').akv({project_id:'3fa218e5-7196-4020-8b30-e2127847cc4f', name:'a.txt', client}) + + +client = await require('@cocalc/backend/conat').conat(); kv = await require('@cocalc/backend/conat/sync').dkv({project_id:'3fa218e5-7196-4020-8b30-e2127847cc4f', name:'a1', client}) + + +client = await require('@cocalc/backend/conat').conat(); s = await require('@cocalc/conat/sync/core-stream').cstream({name:'d.txt',client}) + + +*/ + +import { type Client, ConatError } from "@cocalc/conat/core/client"; +import { + type ConatSocketServer, + type ServerSocket, +} from "@cocalc/conat/socket"; +import { getLogger } from "@cocalc/conat/client"; +import type { + StoredMessage, + PersistentStream, + StorageOptions, +} from "./storage"; +import { getStream, SERVICE, MAX_PER_USER, MAX_GLOBAL, RESOURCE } from "./util"; +import { throttle } from "lodash"; +import { type SetOptions } from "./client"; +import { once } from "@cocalc/util/async-utils"; +import { UsageMonitor } from "@cocalc/conat/monitor/usage"; + +const logger = getLogger("persist:server"); + +// When sending a large number of message for +// getAll or change updates, we combine together messages +// until hitting this size, then send them all at once. +// This bound is to avoid potentially using a huge amount of RAM +// when streaming a large saved database to the client. +// Note: if a single message is larger than this, it still +// gets sent, just individually. +const DEFAULT_MESSAGES_THRESH = 20 * 1e6; +//const DEFAULT_MESSAGES_THRESH = 1e5; + +// I added an experimental way to run any sqlite query... but it is disabled +// since of course there are major DOS and security concerns. +const ENABLE_SQLITE_GENERAL_QUERIES = false; + +const SEND_THROTTLE = 30; + +export function server({ + client, + messagesThresh = DEFAULT_MESSAGES_THRESH, +}: { + client: Client; + messagesThresh?: number; +}) { + logger.debug("server: creating..."); + if (client == null) { + throw Error("client must be specified"); + } + const subject = `${SERVICE}.*`; + const server: ConatSocketServer = client.socket.listen(subject); + logger.debug("server: listening on ", { subject }); + const usage = new UsageMonitor({ + maxPerUser: MAX_PER_USER, + max: MAX_GLOBAL, + resource: RESOURCE, + log: (...args) => { + logger.debug(RESOURCE, ...args); + }, + }); + server.on("close", () => { + usage.close(); + }); + + server.on("connection", (socket: ServerSocket) => { + logger.debug("server: got new connection", { + id: socket.id, + subject: socket.subject, + }); + let error = ""; + let errorCode: any = undefined; + let changefeed = false; + let storage: undefined | StorageOptions = undefined; + let stream: undefined | PersistentStream = undefined; + let user = ""; + let added = false; + socket.on("data", async (data) => { + // logger.debug("server: got data ", data); + if (stream == null) { + storage = data.storage; + changefeed = data.changefeed; + try { + user = socket.subject.split(".")[1]; + usage.add(user); + added = true; + stream = await getStream({ + subject: socket.subject, + storage, + }); + if (changefeed) { + startChangefeed({ socket, stream, messagesThresh }); + } + socket.emit("stream-initialized"); + } catch (err) { + error = `${err}`; + errorCode = err.code; + socket.write(null, { headers: { error, code: errorCode } }); + } + } + }); + socket.on("closed", () => { + logger.debug("socket closed", socket.subject); + storage = undefined; + stream?.close(); + stream = undefined; + if (added) { + usage.delete(user); + } + }); + + socket.on("request", async (mesg) => { + const request = mesg.headers; + // logger.debug("got request", request); + + try { + if (error) { + throw new ConatError(error, { code: errorCode }); + } + if (stream == null) { + await once(socket, "stream-initialized", request.timeout ?? 30000); + } + if (stream == null) { + throw Error("bug"); + } + if (request.cmd == "set") { + mesg.respondSync( + stream.set({ + key: request.key, + previousSeq: request.previousSeq, + raw: mesg.raw, + ttl: request.ttl, + encoding: mesg.encoding, + headers: request.headers, + msgID: request.msgID, + }), + ); + } else if (request.cmd == "setMany") { + // just like set except the main data of the mesg + // has an array of set operations + const resp: ( + | { seq: number; time: number } + | { error: string; code?: any } + )[] = []; + for (const { + key, + previousSeq, + ttl, + msgID, + messageData, + } of mesg.data as SetOptions[]) { + try { + resp.push( + stream.set({ + key, + previousSeq, + ttl, + headers: messageData.headers, + msgID, + raw: messageData.raw, + encoding: messageData.encoding, + }), + ); + } catch (err) { + resp.push({ error: `${err}`, code: err.code }); + } + } + mesg.respondSync(resp); + } else if (request.cmd == "delete") { + mesg.respondSync(stream.delete(request)); + } else if (request.cmd == "config") { + mesg.respondSync(stream.config(request.config)); + } else if (request.cmd == "inventory") { + mesg.respondSync(stream.inventory()); + } else if (request.cmd == "get") { + const resp = stream.get({ key: request.key, seq: request.seq }); + //console.log("got resp = ", resp); + if (resp == null) { + mesg.respondSync(null); + } else { + const { raw, encoding, headers, seq, time, key } = resp; + mesg.respondSync(null, { + raw, + encoding, + headers: { ...headers, seq, time, key }, + }); + } + } else if (request.cmd == "keys") { + const resp = stream.keys(); + mesg.respondSync(resp); + } else if (request.cmd == "sqlite") { + if (!ENABLE_SQLITE_GENERAL_QUERIES) { + throw Error("sqlite command not currently supported"); + } + const resp = stream.sqlite(request.statement, request.params); + mesg.respondSync(resp); + } else if (request.cmd == "serverId") { + mesg.respondSync(server.id); + } else if (request.cmd == "getAll") { + logger.debug("getAll", { subject: socket.subject, request }); + // getAll uses requestMany which responds with all matching messages, + // so no call to mesg.respond here. + getAll({ stream, mesg, request, messagesThresh }); + } else if (request.cmd == "changefeed") { + logger.debug("changefeed", changefeed); + if (!changefeed) { + changefeed = true; + startChangefeed({ socket, stream, messagesThresh }); + } + mesg.respondSync("created"); + } else { + mesg.respondSync(null, { + headers: { error: `unknown command ${request.cmd}`, code: 404 }, + }); + } + } catch (err) { + mesg.respondSync(null, { + headers: { error: `${err}`, code: err.code }, + }); + } + }); + }); + + return server; +} + +async function getAll({ stream, mesg, request, messagesThresh }) { + let seq = 0; + const respond = (error?, messages?: StoredMessage[]) => { + mesg.respondSync(messages, { headers: { error, seq, code: error?.code } }); + seq += 1; + }; + + try { + const messages: StoredMessage[] = []; + let size = 0; + for (const message of stream.getAll({ + start_seq: request.start_seq, + end_seq: request.end_seq, + })) { + messages.push(message); + size += message.raw.length; + if (size >= messagesThresh) { + respond(undefined, messages); + messages.length = 0; + size = 0; + } + } + + if (messages.length > 0) { + respond(undefined, messages); + } + // successful finish + respond(); + } catch (err) { + respond(`${err}`); + } +} + +function startChangefeed({ socket, stream, messagesThresh }) { + logger.debug("startChangefeed", { subject: socket.subject }); + let seq = 0; + const respond = (error?, messages?: StoredMessage[]) => { + if (socket.state == "closed") { + return; + } + //logger.debug("changefeed: writing messages to socket", { seq, messages }); + socket.write(messages, { headers: { error, seq } }); + seq += 1; + }; + + const unsentMessages: StoredMessage[] = []; + const sendAllUnsentMessages = throttle( + () => { + while (socket.state != "closed" && unsentMessages.length > 0) { + const messages: StoredMessage[] = []; + let size = 0; + while (unsentMessages.length > 0 && socket.state != "closed") { + const message = unsentMessages.shift(); + // e.g. op:'delete' messages have length 0 and no raw field + size += message?.raw?.length ?? 0; + messages.push(message!); + if (size >= messagesThresh) { + respond(undefined, messages); + size = 0; + messages.length = 0; + } + } + if (messages.length > 0) { + respond(undefined, messages); + } + } + }, + SEND_THROTTLE, + { leading: true, trailing: true }, + ); + + stream.on("change", (message) => { + if (socket.state == "closed") { + return; + } + //console.log("stream change event", message); + // logger.debug("changefeed got message", message, socket.state); + unsentMessages.push(message); + sendAllUnsentMessages(); + }); +} diff --git a/src/packages/conat/persist/storage.ts b/src/packages/conat/persist/storage.ts new file mode 100644 index 0000000000..421df3728f --- /dev/null +++ b/src/packages/conat/persist/storage.ts @@ -0,0 +1,796 @@ +/* +Persistent storage of a specific stream or kv store. + +You can set a message by providing optionally a key, buffer and/or json value. +A sequence number and time (in ms since epoch) is assigned and returned. +If the key is provided, it is an arbitrary string and all older messages +with that same key are deleted. You can efficiently retrieve a message +by its key. The message content itself is given by the buffer and/or json +value. The buffer is like the "payload" in NATS, and the json is like +the headers in NATS. + +This module is: + + - efficient -- buffer is automatically compressed using zstandard + - synchronous -- fast enough to meet our requirements even with blocking + - memory efficient -- nothing in memory beyond whatever key you request + +We care about memory efficiency here since it's likely we'll want to have +possibly thousands of these in a single nodejs process at once, but with +less than 1 read/write per second for each. Thus memory is critical, and +supporting at least 1000 writes/second is what we need. +Fortunately, this implementation can do ~50,000+ writes per second and read +over 500,000 per second. Yes, it blocks the main thread, but by using +better-sqlite3 and zstd-napi, we get 10x speed increases over async code, +so this is worth it. + + +COMPRESSION: + +I implemented *sync* lz4-napi compression here and it's very fast, +but it has to be run with async waits in a loop or it doesn't give back +memory, and such throttling may significantly negatively impact performance +and mean we don't get a 100% sync api (like we have now). +The async functions in lz4-napi seem fine. Upstream report (by me): +https://github.com/antoniomuso/lz4-napi/issues/678 +I also tried the rust sync snappy and it had a similar memory leak. Finally, +I tried zstd-napi and it has a very fast sync implementation that does *not* +need async pauses to not leak memory. So zstd-napi it is. +And I like zstandard anyways. + +NOTE: + +We use seconds instead of ms in sqlite since that is the standard +convention for times in sqlite. + +DEVELOPMENT: + + + s = require('@cocalc/backend/conat/persist').pstream({path:'/tmp/a.db'}) + +*/ + +import { refCacheSync } from "@cocalc/util/refcache"; +import { createDatabase, type Database, compress, decompress } from "./context"; +import type { JSONValue } from "@cocalc/util/types"; +import { EventEmitter } from "events"; +import { + DataEncoding, + type Headers, + ConatError, +} from "@cocalc/conat/core/client"; +import TTL from "@isaacs/ttlcache"; +import { getLogger } from "@cocalc/conat/client"; + +const logger = getLogger("persist:storage"); + +export interface PartialInventory { + // how much space is used by this stream + bytes: number; + limits: Partial; + // number of messages + count: number; + // for streams, the seq number up to which this data is valid, i.e., + // this data is for all elements of the stream with sequence + // number <= seq. + seq: number; +} + +export interface Configuration { + // How many messages may be in a Stream, oldest messages will be removed + // if the Stream exceeds this size. -1 for unlimited. + max_msgs: number; + + // Maximum age of any message in the stream, + // expressed in milliseconds. 0 for unlimited. + // **Note that max_age is in milliseconds.** + max_age: number; + + // How big the Stream may be. When the stream size + // exceeds this, old messages are removed. -1 for unlimited. + // The size of a message is the sum of the raw uncompressed blob + // size, the headers json and the key length. + max_bytes: number; + + // The largest message that will be accepted. -1 for unlimited. + max_msg_size: number; + + // Attempting to publish a message that causes either of the following + // two rate limits to be exceeded throws an exception. + // For dstream, the messages are explicitly rejected and the client + // gets a "reject" event emitted. E.g., the terminal running in the project + // writes [...] when it gets these rejects, indicating that data was dropped. + // -1 for unlimited + max_bytes_per_second: number; + + // -1 for unlimited + max_msgs_per_second: number; + + // old = delete old messages to make room for nw + // new = refuse writes if they exceed the limits + discard_policy: "old" | "new"; + + // If true (default: false), messages will be automatically deleted after their ttl + // Use the option {ttl:number of MILLISECONDS} when publishing to set a ttl. + allow_msg_ttl: boolean; + + // description of this table + desc: JSONValue; +} + +const CONFIGURATION = { + max_msgs: { def: -1, fromDb: parseInt, toDb: (x) => `${parseInt(x)}` }, + max_age: { def: 0, fromDb: parseInt, toDb: (x) => `${parseInt(x)}` }, + max_bytes: { def: -1, fromDb: parseInt, toDb: (x) => `${parseInt(x)}` }, + max_msg_size: { def: -1, fromDb: parseInt, toDb: (x) => `${parseInt(x)}` }, + max_bytes_per_second: { + def: -1, + fromDb: parseInt, + toDb: (x) => `${parseInt(x)}`, + }, + max_msgs_per_second: { + def: -1, + fromDb: parseInt, + toDb: (x) => `${parseInt(x)}`, + }, + discard_policy: { + def: "old", + fromDb: (x) => `${x}`, + toDb: (x) => (x == "new" ? "new" : "old"), + }, + allow_msg_ttl: { + def: false, + fromDb: (x) => x == "true", + toDb: (x) => `${!!x}`, + }, + desc: { + def: null, + fromDb: JSON.parse, + toDb: JSON.stringify, + }, +}; + +export const EPHEMERAL_MAX_BYTES = 64 * 1e6; + +enum CompressionAlgorithm { + None = 0, + Zstd = 1, +} + +interface Compression { + // compression algorithm to use + algorithm: CompressionAlgorithm; + // only compress data above this size + threshold: number; +} + +const DEFAULT_COMPRESSION = { + algorithm: CompressionAlgorithm.Zstd, + threshold: 1024, +}; + +export interface StoredMessage { + // server assigned positive increasing integer number + seq: number; + // server assigned time in ms since epoch + time: number; + // user assigned key -- when set all previous messages with that key are deleted. + key?: string; + // the encoding used to encode the raw data + encoding: DataEncoding; + // arbitrary binary data + raw: Buffer; + // arbitrary JSON-able object -- analogue of NATS headers, but anything JSON-able + headers?: Headers; +} + +export interface SetOperation extends StoredMessage { + op: undefined; + msgID?: string; +} + +export interface DeleteOperation { + op: "delete"; + // sequence numbers of deleted messages + seqs: number[]; +} + +export interface StorageOptions { + // absolute path to sqlite database file. This needs to be a valid filename + // path, and must also be kept under 1K so it can be stored in cloud storage. + path: string; + // if false (the default) do not require sync writes to disk on every set + sync?: boolean; + // if set, then data is never saved to disk at all. To avoid using a lot of server + // RAM there is always a hard cap of at most EPHEMERAL_MAX_BYTES on any ephemeral + // table, which is enforced on all writes. Clients should always set max_bytes, + // possibly as low as they can, and check by reading back what is set. + ephemeral?: boolean; + // compression configuration + compression?: Compression; +} + +// persistence for stream of messages with subject +export class PersistentStream extends EventEmitter { + private readonly options: StorageOptions; + private readonly db: Database; + private readonly msgIDs = new TTL({ ttl: 2 * 60 * 1000 }); + private conf: Configuration; + + constructor(options: StorageOptions) { + super(); + logger.debug("constructor ", options.path); + + this.setMaxListeners(1000); + options = { compression: DEFAULT_COMPRESSION, ...options }; + this.options = options; + const location = this.options.ephemeral + ? ":memory:" + : this.options.path + ".db"; + this.db = createDatabase(location); + //console.log(location); + this.init(); + } + + init = () => { + if (!this.options.sync && !this.options.ephemeral) { + // Unless sync is set, we do not require that the filesystem has commited changes + // to disk after every insert. This can easily make things 10x faster. sets are + // typically going to come in one-by-one as users edit files, so this works well + // for our application. Also, loss of a few seconds persistence is acceptable + // in a lot of applications, e.g., if it is just edit history for a file. + this.db.prepare("PRAGMA synchronous=OFF").run(); + } + // time is in *seconds* since the epoch, since that is standard for sqlite. + // ttl is in milliseconds. + this.db + .prepare( + `CREATE TABLE IF NOT EXISTS messages ( + seq INTEGER PRIMARY KEY AUTOINCREMENT, key TEXT UNIQUE, time INTEGER NOT NULL, headers TEXT, compress NUMBER NOT NULL, encoding NUMBER NOT NULL, raw BLOB NOT NULL, size NUMBER NOT NULL, ttl NUMBER + ) + `, + ) + .run(); + this.db + .prepare( + ` + CREATE TABLE IF NOT EXISTS config ( + field TEXT PRIMARY KEY, value TEXT NOT NULL + )`, + ) + .run(); + this.db + .prepare("CREATE INDEX IF NOT EXISTS idx_messages_key ON messages(key)") + .run(); + this.db + .prepare("CREATE INDEX IF NOT EXISTS idx_messages_time ON messages(time)") + .run(); + + this.conf = this.config(); + }; + + close = () => { + logger.debug("close ", this.options.path); + if (this.db != null) { + this.vacuum(); + this.db.prepare("PRAGMA wal_checkpoint(FULL)").run(); + this.db.close(); + // @ts-ignore + } + // @ts-ignore + delete this.options; + this.msgIDs?.clear(); + // @ts-ignore + delete this.msgIDs; + }; + + private compress = ( + raw: Buffer, + ): { raw: Buffer; compress: CompressionAlgorithm } => { + if ( + this.options.compression!.algorithm == CompressionAlgorithm.None || + raw.length <= this.options.compression!.threshold + ) { + return { raw, compress: CompressionAlgorithm.None }; + } + if (this.options.compression!.algorithm == CompressionAlgorithm.Zstd) { + return { raw: compress(raw), compress: CompressionAlgorithm.Zstd }; + } + throw Error( + `unknown compression algorithm: ${this.options.compression!.algorithm}`, + ); + }; + + set = ({ + encoding, + raw, + headers, + key, + ttl, + previousSeq, + msgID, + }: { + encoding: DataEncoding; + raw: Buffer; + headers?: JSONValue; + key?: string; + ttl?: number; + previousSeq?: number; + // if given, any attempt to publish something again with the same msgID + // is deduplicated. Use this to prevent accidentally writing twice, e.g., + // due to not getting a response back from the server. + msgID?: string; + }): { seq: number; time: number } => { + if (previousSeq === null) { + previousSeq = undefined; + } + if (key === null) { + key = undefined; + } + if (msgID != null && this.msgIDs?.has(msgID)) { + return this.msgIDs.get(msgID)!; + } + if (key !== undefined && previousSeq !== undefined) { + // throw error if current seq number for the row + // with this key is not previousSeq. + const { seq } = this.db // there is an index on the key so this is fast + .prepare("SELECT seq FROM messages WHERE key=?") + .get(key) as any; + if (seq != previousSeq) { + throw new ConatError("wrong last sequence", { + code: "wrong-last-sequence", + }); + } + } + const time = Date.now(); + const compressedRaw = this.compress(raw); + const serializedHeaders = JSON.stringify(headers); + const size = + (serializedHeaders?.length ?? 0) + + (raw?.length ?? 0) + + (key?.length ?? 0); + + this.enforceLimits(size); + + const tx = this.db.transaction( + (time, compress, encoding, raw, headers, key, size, ttl) => { + if (key !== undefined) { + // insert with key -- delete all previous messages, as they will + // never be needed again and waste space. + this.db.prepare("DELETE FROM messages WHERE key = ?").run(key); + } + return this.db + .prepare( + "INSERT INTO messages(time, compress, encoding, raw, headers, key, size, ttl) VALUES (?, ?, ?, ?, ?, ?, ?, ?) RETURNING seq", + ) + .get(time / 1000, compress, encoding, raw, headers, key, size, ttl); + }, + ); + const row = tx( + time, + compressedRaw.compress, + encoding, + compressedRaw.raw, + serializedHeaders, + key, + size, + ttl, + ); + const seq = Number((row as any).seq); + // lastInsertRowid - is a bigint from sqlite, but we won't hit that limit + this.emit("change", { + op: "set", + seq, + time, + key, + encoding, + raw, + headers, + msgID, + }); + if (msgID !== undefined) { + this.msgIDs.set(msgID, { time, seq }); + } + return { time, seq }; + }; + + get = ({ + seq, + key, + }: { seq: number; key: undefined } | { seq: undefined; key: string }): + | StoredMessage + | undefined => { + let x; + if (seq) { + x = this.db + .prepare( + "SELECT seq, key, time, compress, encoding, raw, headers FROM messages WHERE seq=?", + ) + .get(seq); + } else if (key != null) { + // NOTE: we guarantee when doing set above that there is at most one + // row with a given key. Also there's a unique constraint. + x = this.db + .prepare( + "SELECT seq, key, time, compress, encoding, raw, headers FROM messages WHERE key=?", + ) + .get(key); + } else { + x = undefined; + } + return dbToMessage(x as any); + }; + + *getAll({ + start_seq, + end_seq, + }: { + end_seq?: number; + start_seq?: number; + } = {}): IterableIterator { + let query: string, stmt; + + const where: string[] = []; + const v: number[] = []; + if (start_seq != null) { + where.push("seq>=?"); + v.push(start_seq); + } + if (end_seq != null) { + where.push("seq<=?"); + v.push(end_seq); + } + query = `SELECT seq, key, time, compress, encoding, raw, headers FROM messages ${where.length == 0 ? "" : " where " + where.join(" AND ")} ORDER BY seq`; + stmt = this.db.prepare(query); + for (const row of stmt.iterate(...v)) { + yield dbToMessage(row)!; + } + } + + delete = ({ + seq, + last_seq, + all, + }: { + seq?: number; + last_seq?: number; + all?: boolean; + }): { seqs: number[] } => { + let seqs: number[] = []; + if (all) { + seqs = this.db + .prepare("SELECT seq FROM messages") + .all() + .map((row: any) => row.seq); + this.db.prepare("DELETE FROM messages").run(); + this.vacuum(); + } else if (last_seq) { + seqs = this.db + .prepare("SELECT seq FROM messages WHERE seq<=?") + .all(last_seq) + .map((row: any) => row.seq); + this.db.prepare("DELETE FROM messages WHERE seq<=?").run(last_seq); + this.vacuum(); + } else if (seq) { + seqs = this.db + .prepare("SELECT seq FROM messages WHERE seq=?") + .all(seq) + .map((row: any) => row.seq); + this.db.prepare("DELETE FROM messages WHERE seq=?").run(seq); + } + this.emit("change", { op: "delete", seqs }); + return { seqs }; + }; + + vacuum = () => { + try { + this.db.prepare("VACUUM").run(); + } catch {} + }; + + get length(): number { + const { length } = this.db + .prepare("SELECT COUNT(*) AS length FROM messages") + .get() as { length: number }; + return length; + } + + totalSize = (): number => { + return ( + (this.db.prepare(`SELECT SUM(size) AS sum FROM messages`).get() as any) + .sum ?? 0 + ); + }; + + seq = (): number => { + return ( + (this.db.prepare(`SELECT MAX(seq) AS seq FROM messages`).get() as any) + .seq ?? 0 + ); + }; + + inventory = (): PartialInventory => { + return { + bytes: this.totalSize(), + count: this.length, + limits: this.getConfig(), + seq: this.seq(), + }; + }; + + keys = (): string[] => { + const v = this.db + .prepare("SELECT key FROM messages WHERE key IS NOT NULL") + .all() as { + key: string; + }[]; + return v.map(({ key }) => key); + }; + + sqlite = (statement: string, params: any[] = []): any[] => { + // Matches "attach database" (case-insensitive, ignores whitespace) + if (/\battach\s+database\b/i.test(statement)) { + throw Error("ATTACH DATABASE not allowed"); + } + const stmt = this.db.prepare(statement); + try { + return stmt.all(...params); + } catch (err) { + if (err.message.includes("run() instead")) { + stmt.run(...params); + return []; + } else { + throw err; + } + } + }; + + // only returns fields that are not set to their default value, + // and doesn't enforce any limits + getConfig = (): Partial => { + const cur: any = {}; + for (const { field, value } of this.db + .prepare("SELECT * FROM config") + .all() as any) { + const { def, fromDb } = CONFIGURATION[field]; + cur[field] = fromDb(value); + if (cur[field] == def) { + delete cur[field]; + } + } + return cur; + }; + + config = (config?: Partial): Configuration => { + const cur: any = {}; + for (const { field, value } of this.db + .prepare("SELECT * FROM config") + .all() as any) { + cur[field] = value; + } + const full: Partial = {}; + for (const key in CONFIGURATION) { + const { def, fromDb, toDb } = CONFIGURATION[key]; + full[key] = + config?.[key] ?? (cur[key] !== undefined ? fromDb(cur[key]) : def); + let x = toDb(full[key]); + if (config?.[key] != null && full[key] != (cur[key] ?? def)) { + // making a change + this.db + .prepare( + `INSERT INTO config (field, value) VALUES(?, ?) ON CONFLICT(field) DO UPDATE SET value=excluded.value`, + ) + .run(key, x); + } + full[key] = fromDb(x); + if ( + this.options.ephemeral && + key == "max_bytes" && + (full[key] == null || full[key] <= 0 || full[key] > EPHEMERAL_MAX_BYTES) + ) { + // for ephemeral we always make it so max_bytes is capped + // (note -- this isn't explicitly set in the sqlite database, since we might + // change it, and by not setting it in the database we can) + full[key] = EPHEMERAL_MAX_BYTES; + } + } + this.conf = full as Configuration; + // ensure any new limits are enforced + this.enforceLimits(0); + return full as Configuration; + }; + + private emitDelete = (rows) => { + if (rows.length > 0) { + const seqs = rows.map((row: { seq: number }) => row.seq); + this.emit("change", { op: "delete", seqs }); + } + }; + + // do whatever limit enforcement and throttling is needed when inserting one new message + // with the given size; if size=0 assume not actually inserting a new message, and just + // enforcingt current limits + private enforceLimits = (size: number = 0) => { + if ( + size > 0 && + (this.conf.max_msgs_per_second > 0 || this.conf.max_bytes_per_second > 0) + ) { + const { msgs, bytes } = this.db + .prepare( + "SELECT COUNT(*) AS msgs, SUM(size) AS bytes FROM messages WHERE time >= ?", + ) + .get(Date.now() / 1000 - 1) as { msgs: number; bytes: number }; + if ( + this.conf.max_msgs_per_second > 0 && + msgs > this.conf.max_msgs_per_second + ) { + throw new ConatError("max_msgs_per_second exceeded", { + code: "reject", + }); + } + if ( + this.conf.max_bytes_per_second > 0 && + bytes > this.conf.max_bytes_per_second + ) { + throw new ConatError("max_bytes_per_second exceeded", { + code: "reject", + }); + } + } + + if (this.conf.max_msgs > -1) { + const length = this.length + (size > 0 ? 1 : 0); + if (length > this.conf.max_msgs) { + if (this.conf.discard_policy == "new") { + if (size > 0) { + throw new ConatError("max_msgs limit reached", { code: "reject" }); + } + } else { + // delete earliest messages to make room + const rows = this.db + .prepare( + `DELETE FROM messages WHERE seq IN (SELECT seq FROM messages ORDER BY seq ASC LIMIT ?) RETURNING seq`, + ) + .all(length - this.conf.max_msgs); + this.emitDelete(rows); + } + } + } + + if (this.conf.max_age > 0) { + const rows = this.db + .prepare( + `DELETE FROM messages WHERE seq IN (SELECT seq FROM messages WHERE time <= ?) RETURNING seq`, + ) + .all((Date.now() - this.conf.max_age) / 1000); + this.emitDelete(rows); + } + + if (this.conf.max_bytes > -1) { + if (size > this.conf.max_bytes) { + if (this.conf.discard_policy == "new") { + if (size > 0) { + throw new ConatError("max_bytes limit reached", { code: "reject" }); + } + } else { + // new message exceeds total, so this is the same as adding in the new message, + // then deleting everything. + this.delete({ all: true }); + } + } else { + // delete all the earliest (in terms of seq number) messages + // so that the sum of the remaining + // sizes along with the new size is <= max_bytes. + // Only enforce if actually inserting, or if current sum is over + const totalSize = this.totalSize(); + const newTotal = totalSize + size; + if (newTotal > this.conf.max_bytes) { + const bytesToFree = newTotal - this.conf.max_bytes; + let freed = 0; + let lastSeqToDelete: number | null = null; + + for (const { seq, size: msgSize } of this.db + .prepare(`SELECT seq, size FROM messages ORDER BY seq ASC`) + .iterate() as any) { + if (freed >= bytesToFree) break; + freed += msgSize; + lastSeqToDelete = seq; + } + + if (lastSeqToDelete !== null) { + if (this.conf.discard_policy == "new") { + if (size > 0) { + throw new ConatError("max_bytes limit reached", { + code: "reject", + }); + } + } else { + const rows = this.db + .prepare(`DELETE FROM messages WHERE seq <= ? RETURNING seq`) + .all(lastSeqToDelete); + this.emitDelete(rows); + } + } + } + } + } + + if (this.conf.allow_msg_ttl) { + const rows = this.db + .prepare( + `DELETE FROM messages WHERE ttl IS NOT null AND time + ttl/1000 < ? RETURNING seq`, + ) + .all(Date.now() / 1000); + this.emitDelete(rows); + } + + if (this.conf.max_msg_size > -1 && size > this.conf.max_msg_size) { + throw new ConatError( + `max_msg_size of ${this.conf.max_msg_size} bytes exceeded`, + { code: "reject" }, + ); + } + }; +} + +function dbToMessage( + x: + | { + seq: number; + key?: string; + time: number; + compress: CompressionAlgorithm; + encoding: DataEncoding; + raw: Buffer; + headers?: string; + } + | undefined, +): StoredMessage | undefined { + if (x === undefined) { + return x; + } + return { + seq: x.seq, + time: x.time * 1000, + key: x.key != null ? x.key : undefined, + encoding: x.encoding, + raw: handleDecompress(x), + headers: x.headers ? JSON.parse(x.headers) : undefined, + }; +} + +function handleDecompress({ + raw, + compress, +}: { + raw: Buffer; + compress: CompressionAlgorithm; +}) { + if (compress == CompressionAlgorithm.None) { + return raw; + } else if (compress == CompressionAlgorithm.Zstd) { + return decompress(raw); + } else { + throw Error(`unknown compression ${compress}`); + } +} + +interface CreateOptions extends StorageOptions { + noCache?: boolean; +} + +export const cache = refCacheSync({ + name: "persistent-storage-stream", + createKey: ({ path }: CreateOptions) => path, + createObject: (options: CreateOptions) => { + const pstream = new PersistentStream(options); + pstream.init(); + return pstream; + }, +}); + +export function pstream( + options: StorageOptions & { noCache?: boolean }, +): PersistentStream { + return cache(options); +} diff --git a/src/packages/conat/persist/util.ts b/src/packages/conat/persist/util.ts new file mode 100644 index 0000000000..44477b2729 --- /dev/null +++ b/src/packages/conat/persist/util.ts @@ -0,0 +1,92 @@ +/* + +Maybe storage available as a service. + +This code is similar to the changefeed server, because +it provides a changefeed on a given persist storage, +and a way to see values. + +DEVELOPMENT: + +Change to the packages/backend directory and run node. + +TERMINAL 1: This sets up the environment and starts the server running: + + require('@cocalc/backend/conat/persist').initServer() + + +TERMINAL 2: In another node session, create a client: + + user = {account_id:'00000000-0000-4000-8000-000000000000'}; storage = {path:'a.db'}; const {id, stream} = await require('@cocalc/backend/conat/persist').getAll({user, storage}); console.log({id}); for await(const x of stream) { console.log(x.data) }; console.log("DONE") + +// client also does this periodically to keep subscription alive: + + await renew({user, id }) + +TERMINAL 3: + +user = {account_id:'00000000-0000-4000-8000-000000000000'}; storage = {path:'a.db'}; const {set,get} = require('@cocalc/backend/conat/persist'); const { messageData } =require("@cocalc/conat/core/client"); 0; + + await set({user, storage, messageData:messageData('hi')}) + + await get({user, storage, seq:1}) + + await set({user, storage, key:'bella', messageData:messageData('hi', {headers:{x:10}})}) + + await get({user, storage, key:'bella'}) + +Also getAll using start_seq: + + cf = const {id, stream} = await require('@cocalc/backend/conat/persist').getAll({user, storage, start_seq:10}); for await(const x of stream) { console.log(x) }; +*/ + +import { assertHasWritePermission } from "./auth"; +import { pstream, PersistentStream } from "./storage"; +import { join } from "path"; +import { syncFiles, ensureContainingDirectoryExists } from "./context"; + +// this is per-server -- and "user" means where the resource is, usually +// a given project. E.g., 500 streams in a project, across many users. +export const MAX_PER_USER = 500; +export const MAX_GLOBAL = 10000; +export const RESOURCE = "persistent storage"; + +//import { getLogger } from "@cocalc/conat/client"; +//const logger = getLogger("persist:util"); + +export const SERVICE = "persist"; + +export type User = { + account_id?: string; + project_id?: string; + hub_id?: string; +}; + +export function persistSubject({ account_id, project_id }: User) { + if (account_id) { + return `${SERVICE}.account-${account_id}`; + } else if (project_id) { + return `${SERVICE}.project-${project_id}`; + } else { + return `${SERVICE}.hub`; + } +} + +export async function getStream({ + subject, + storage, +}): Promise { + // this permsissions check should always work and use should + // never see an error here, since + // the same check runs on the client before the message is sent to the + // persist storage. However a malicious user would hit this. + // IMPORTANT: must be here so error *code* also sent back. + assertHasWritePermission({ + subject, + path: storage.path, + }); + const path = join(syncFiles.local, storage.path); + await ensureContainingDirectoryExists(path); + return pstream({ ...storage, path }); +} + diff --git a/src/packages/nats/project-api/editor.ts b/src/packages/conat/project/api/editor.ts similarity index 72% rename from src/packages/nats/project-api/editor.ts rename to src/packages/conat/project/api/editor.ts index f90c9079cb..cd70dc973d 100644 --- a/src/packages/nats/project-api/editor.ts +++ b/src/packages/conat/project/api/editor.ts @@ -11,8 +11,21 @@ export const editor = { jupyterKernelLogo: true, jupyterKernels: true, formatterString: true, + printSageWS: true, + createTerminalService: true, }; +export interface CreateTerminalOptions { + env?: { [key: string]: string }; + command?: string; + args?: string[]; + cwd?: string; + ephemeral?: boolean; + // path of the primary tab in the browser, e.g., if you open a.term it's a.term for all frames, + // and if you have term next to a.md (say), then it is a.md. + path: string; +} + export interface Editor { // Create a new file with the given name, possibly aware of templates. // This was cc-new-file in the old smc_pyutils python library. This @@ -39,4 +52,11 @@ export interface Editor { options: FormatterOptions; path?: string; // only used for CLANG }) => Promise; + + printSageWS: (opts) => Promise; + + createTerminalService: ( + termPath: string, + opts: CreateTerminalOptions, + ) => Promise; } diff --git a/src/packages/nats/project-api/index.ts b/src/packages/conat/project/api/index.ts similarity index 93% rename from src/packages/nats/project-api/index.ts rename to src/packages/conat/project/api/index.ts index 2bc684cc0f..694229b2bc 100644 --- a/src/packages/nats/project-api/index.ts +++ b/src/packages/conat/project/api/index.ts @@ -1,7 +1,7 @@ import { type System, system } from "./system"; import { type Editor, editor } from "./editor"; import { type Sync, sync } from "./sync"; -import { handleErrorMessage } from "@cocalc/nats/util"; +import { handleErrorMessage } from "@cocalc/conat/util"; export interface ProjectApi { system: System; diff --git a/src/packages/nats/project-api/sync.ts b/src/packages/conat/project/api/sync.ts similarity index 100% rename from src/packages/nats/project-api/sync.ts rename to src/packages/conat/project/api/sync.ts diff --git a/src/packages/nats/project-api/system.ts b/src/packages/conat/project/api/system.ts similarity index 77% rename from src/packages/nats/project-api/system.ts rename to src/packages/conat/project/api/system.ts index 74db7fa299..3e780381c8 100644 --- a/src/packages/nats/project-api/system.ts +++ b/src/packages/conat/project/api/system.ts @@ -7,10 +7,10 @@ import type { Configuration, ConfigurationAspect, } from "@cocalc/comm/project-configuration"; +import { type ProjectJupyterApiOptions } from "@cocalc/util/jupyter/api-types"; export const system = { terminate: true, - resetConnection: true, version: true, @@ -21,7 +21,7 @@ export const system = { realpath: true, canonicalPaths: true, - // these should be completel deprecated -- the new streaming writeFile and readFile in nats/files are much better. + // these should be deprecated -- the new streaming writeFile and readFile in conat/files are better. writeTextFileToProject: true, readTextFileFromProject: true, @@ -31,18 +31,15 @@ export const system = { exec: true, signal: true, + + // jupyter stateless API + jupyterExecute: true, }; export interface System { // stop the api service terminate: () => Promise; - // close the nats connection -- this is meant for development purposes - // and closes the connection; the connection monitor should then reoopen it within - // a few seconds. This is, of course, likely to NOT return, since the - // connection is broken for a bit. - resetConnection: () => Promise<{ closed: boolean }>; - version: () => Promise; listing: (opts: { @@ -75,4 +72,6 @@ export interface System { pids?: number[]; pid?: number; }) => Promise; + + jupyterExecute: (opts: ProjectJupyterApiOptions) => Promise; } diff --git a/src/packages/conat/project/project-info.ts b/src/packages/conat/project/project-info.ts new file mode 100644 index 0000000000..ac63263a93 --- /dev/null +++ b/src/packages/conat/project/project-info.ts @@ -0,0 +1,85 @@ +/* + * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. + * License: MS-RSL – see LICENSE.md for details + */ + +import type { ProjectInfo } from "@cocalc/util/types/project-info/types"; +export type { ProjectInfo }; +import { getLogger } from "@cocalc/conat/client"; +import { projectSubject } from "@cocalc/conat/names"; +import { conat } from "@cocalc/conat/client"; + +const SERVICE_NAME = "project-info"; +const logger = getLogger("project:project-info"); + +interface Api { + get: () => Promise; +} + +export async function get({ + project_id, + compute_server_id = 0, +}: { + project_id: string; + compute_server_id?: number; +}) { + const c = await conat(); + const subject = getSubject({ project_id, compute_server_id }); + return await c.call(subject).get(); +} + +function getSubject({ project_id, compute_server_id }) { + return projectSubject({ + project_id, + compute_server_id, + service: SERVICE_NAME, + }); +} + +export function createService(opts: { + infoServer; + project_id: string; + compute_server_id: number; +}) { + return new ProjectInfoService(opts); +} + +class ProjectInfoService { + private infoServer?; + private service?; + private readonly subject: string; + info?: ProjectInfo | null = null; + + constructor({ infoServer, project_id, compute_server_id }) { + logger.debug("register"); + this.subject = getSubject({ project_id, compute_server_id }); + // initializing project info server + reacting when it has something to say + this.infoServer = infoServer; + this.infoServer.start(); + this.infoServer.on("info", this.saveInfo); + this.createService(); + } + + private saveInfo = (info) => { + this.info = info; + }; + + private createService = async () => { + logger.debug("started project info service ", { subject: this.subject }); + const client = await conat(); + this.service = await client.service(this.subject, { + get: async () => this.info ?? null, + }); + }; + + close = (): void => { + if (this.infoServer == null) { + return; + } + logger.debug("close"); + this.infoServer?.removeListener("info", this.saveInfo); + delete this.infoServer; + this.service?.close(); + delete this.service; + } +} diff --git a/src/packages/conat/project/project-status.ts b/src/packages/conat/project/project-status.ts new file mode 100644 index 0000000000..f3a34d11d5 --- /dev/null +++ b/src/packages/conat/project/project-status.ts @@ -0,0 +1,44 @@ +/* +Broadcast project status whenever it updates. +*/ + +import { projectSubject } from "@cocalc/conat/names"; +import { conat } from "@cocalc/conat/client"; +import { getLogger } from "@cocalc/conat/client"; +import { type Subscription } from "@cocalc/conat/core/client"; + +const SERVICE_NAME = "project-status"; +const logger = getLogger("project:project-status"); + +function getSubject({ project_id, compute_server_id }) { + return projectSubject({ + project_id, + compute_server_id, + service: SERVICE_NAME, + }); +} + +// publishes status updates when they are emitted. +export async function createPublisher({ + project_id, + compute_server_id, + projectStatusServer, +}) { + const client = await conat(); + const subject = getSubject({ project_id, compute_server_id }); + logger.debug("publishing status updates on ", { subject }); + projectStatusServer.on("status", (status) => { + logger.debug("publishing updated status", status); + client.publishSync(subject, status); + }); +} + +// async iterator over the status updates: +export async function get({ + project_id, + compute_server_id, +}): Promise { + const client = await conat(); + const subject = getSubject({ project_id, compute_server_id }); + return await client.subscribe(subject); +} diff --git a/src/packages/conat/project/usage-info.ts b/src/packages/conat/project/usage-info.ts new file mode 100644 index 0000000000..0b80f00fd6 --- /dev/null +++ b/src/packages/conat/project/usage-info.ts @@ -0,0 +1,106 @@ +/* +Provide info about a specific path, derived from the project-status stream. +E.g. cpu/ram usage by a Jupyter notebook kernel. + +This starts measuring when a request comes in for a path and stop when +there is no request for a while. +*/ + +import { projectSubject } from "@cocalc/conat/names"; +import { conat } from "@cocalc/conat/client"; +import { getLogger } from "@cocalc/conat/client"; +import { type UsageInfo } from "@cocalc/util/types/project-usage-info"; +import TTL from "@isaacs/ttlcache"; + +const logger = getLogger("project:usage-info"); + +type InfoServer = any; + +const SERVICE_NAME = "usage-info"; + +// we automatically stop computing data about a specific path after this amount of +// time elapses with no user requests. Users make a request every 2-3 seconds, +// and even it times out, everything starts again in 2-3 seconds. So this is fine. +const SERVER_TIMEOUT = 15000; + +function getSubject({ project_id, compute_server_id }) { + return projectSubject({ + project_id, + compute_server_id, + service: SERVICE_NAME, + }); +} + +interface Api { + get: (path) => Promise; +} + +export async function get({ + project_id, + compute_server_id = 0, + path, +}: { + project_id: string; + compute_server_id?: number; + path: string; +}) { + const c = await conat(); + const subject = getSubject({ project_id, compute_server_id }); + return await c.call(subject).get(path); +} + +interface Options { + project_id: string; + compute_server_id: number; + createUsageInfoServer: Function; +} + +export class UsageInfoService { + private service?; + private infoServers = new TTL({ + ttl: SERVER_TIMEOUT, + dispose: (server) => this.dispose(server), + }); + private usage = new TTL({ ttl: 2 * SERVER_TIMEOUT }); + + constructor(private options: Options) { + this.createService(); + } + + private createService = async () => { + const subject = getSubject(this.options); + logger.debug("starting usage-info service", { subject }); + const client = await conat(); + this.service = await client.service(subject, { + get: this.get, + }); + }; + + private get = async (path: string): Promise => { + if (!this.infoServers.has(path)) { + logger.debug("creating new usage server for ", { path }); + const server = this.options.createUsageInfoServer(path); + this.infoServers.set(path, server); + server.on("usage", (usage) => { + // logger.debug("got new info", { path, usage }); + this.usage.set(path, usage); + }); + } + return this.usage.get(path) ?? null; + }; + + private dispose = (server) => { + server.close(); + }; + + close = (): void => { + this.infoServers.clear(); + this.usage.clear(); + this.service?.close(); + delete this.service; + }; +} + +export function createUsageInfoService(options: Options) { + return new UsageInfoService(options); +} diff --git a/src/packages/nats/service/formatter.ts b/src/packages/conat/service/formatter.ts similarity index 100% rename from src/packages/nats/service/formatter.ts rename to src/packages/conat/service/formatter.ts diff --git a/src/packages/conat/service/index.ts b/src/packages/conat/service/index.ts new file mode 100644 index 0000000000..fd6e64b000 --- /dev/null +++ b/src/packages/conat/service/index.ts @@ -0,0 +1,8 @@ +export type { + ServiceDescription, + CallConatServiceFunction, + ServiceCall, + CreateConatServiceFunction, + ConatService, +} from "./service"; +export { callConatService, createConatService } from "./service"; diff --git a/src/packages/nats/service/jupyter.ts b/src/packages/conat/service/jupyter.ts similarity index 81% rename from src/packages/nats/service/jupyter.ts rename to src/packages/conat/service/jupyter.ts index 5ea9093395..a581be23b9 100644 --- a/src/packages/nats/service/jupyter.ts +++ b/src/packages/conat/service/jupyter.ts @@ -46,12 +46,6 @@ export interface JupyterApi { export type JupyterApiEndpoint = keyof JupyterApi; -// we use request many for all calls to the Jupyter server, because -// at least one call -- more_output -- is very likely to return -// very large results (it's kind of the point), and this makes -// handling this very easy. -const REQUEST_MANY = true; - export function jupyterApiClient({ project_id, path, @@ -66,11 +60,10 @@ export function jupyterApiClient({ path, service, timeout, - many: REQUEST_MANY, }); } -export async function createNatsJupyterService({ +export async function createConatJupyterService({ path, project_id, impl, @@ -85,6 +78,5 @@ export async function createNatsJupyterService({ service, impl, description: "Jupyter notebook compute API", - many: REQUEST_MANY, }); } diff --git a/src/packages/nats/service/listings.ts b/src/packages/conat/service/listings.ts similarity index 98% rename from src/packages/nats/service/listings.ts rename to src/packages/conat/service/listings.ts index 5213db6975..0d194c5ae9 100644 --- a/src/packages/nats/service/listings.ts +++ b/src/packages/conat/service/listings.ts @@ -4,7 +4,7 @@ Service for watching directory listings in a project or compute server. import { createServiceClient, createServiceHandler } from "./typed"; import type { DirectoryListingEntry } from "@cocalc/util/types"; -import { dkv, type DKV } from "@cocalc/nats/sync/dkv"; +import { dkv, type DKV } from "@cocalc/conat/sync/dkv"; import { EventEmitter } from "events"; import refCache from "@cocalc/util/refcache"; @@ -66,7 +66,7 @@ export async function createListingsService({ }); } -const limits = { +const config = { max_msgs: MAX_DIRECTORIES, }; @@ -84,7 +84,7 @@ export async function getListingsKV( ): Promise> { return await dkv({ name: "listings", - limits, + config, ...opts, }); } @@ -101,7 +101,7 @@ export async function getListingsTimesKV( ): Promise> { return await dkv({ name: "listings-times", - limits, + config, ...opts, }); } diff --git a/src/packages/conat/service/service.ts b/src/packages/conat/service/service.ts new file mode 100644 index 0000000000..760205db90 --- /dev/null +++ b/src/packages/conat/service/service.ts @@ -0,0 +1,282 @@ +/* +Simple to use UI to connect anything in cocalc via request/reply services. + +- callConatService +- createConatService + +The input is basically where the service is (account, project, public), +and either what message to send or how to handle messages. +Also if the handler throws an error, the caller will throw +an error too. +*/ + +import { type Location } from "@cocalc/conat/types"; +import { conat, getLogger } from "@cocalc/conat/client"; +import { randomId } from "@cocalc/conat/names"; +import { EventEmitter } from "events"; +import { encodeBase64 } from "@cocalc/conat/util"; +import { type Client } from "@cocalc/conat/core/client"; +import { until } from "@cocalc/util/async-utils"; + +const DEFAULT_TIMEOUT = 10 * 1000; + +const logger = getLogger("conat:service"); + +export interface ServiceDescription extends Location { + service: string; + + description?: string; + + // if true and multiple servers are setup in same "location", then they ALL get to respond (sender gets first response). + all?: boolean; + + // DEFAULT: ENABLE_SERVICE_FRAMEWORK + enableServiceFramework?: boolean; + + subject?: string; +} + +export interface ServiceCall extends ServiceDescription { + mesg: any; + timeout?: number; + + // if it fails with error.code 503, we wait for service to be ready and try again, + // unless this is set -- e.g., when waiting for the service in the first + // place we set this to avoid an infinite loop. + // This now just uses the waitForInterest option to request. + noRetry?: boolean; + + client?: Client; +} + +export async function callConatService(opts: ServiceCall): Promise { + // console.log("callConatService", opts); + const cn = opts.client ?? (await conat()); + const subject = serviceSubject(opts); + let resp; + const timeout = opts.timeout ?? DEFAULT_TIMEOUT; + // ensure not undefined, since undefined can't be published. + const data = opts.mesg ?? null; + + const doRequest = async () => { + resp = await cn.request(subject, data, { + timeout, + waitForInterest: !opts.noRetry, + }); + const result = resp.data; + if (result?.error) { + throw Error(result.error); + } + return result; + }; + return await doRequest(); +} + +export type CallConatServiceFunction = typeof callConatService; + +export interface Options extends ServiceDescription { + description?: string; + version?: string; + handler: (mesg) => Promise; + client?: Client; +} + +export function createConatService(options: Options) { + return new ConatService(options); +} + +export type CreateConatServiceFunction = typeof createConatService; + +export function serviceSubject({ + service, + + account_id, + browser_id, + + project_id, + compute_server_id, + + path, + + subject, +}: ServiceDescription): string { + if (subject) { + return subject; + } + let segments; + path = path ? encodeBase64(path) : "_"; + if (!project_id && !account_id) { + segments = ["public", service]; + } else if (account_id) { + segments = [ + "services", + `account-${account_id}`, + browser_id ?? "_", + project_id ?? "_", + path ?? "_", + service, + ]; + } else if (project_id) { + segments = [ + "services", + `project-${project_id}`, + compute_server_id ?? "_", + service, + path, + ]; + } + return segments.join("."); +} + +export function serviceName({ + service, + + account_id, + browser_id, + + project_id, + compute_server_id, +}: ServiceDescription): string { + let segments; + if (!project_id && !account_id) { + segments = [service]; + } else if (account_id) { + segments = [`account-${account_id}`, browser_id ?? "-", service]; + } else if (project_id) { + segments = [`project-${project_id}`, compute_server_id ?? "-", service]; + } + return segments.join("-"); +} + +export function serviceDescription({ + description, + path, +}: ServiceDescription): string { + return [description, path ? `\nPath: ${path}` : ""].join(""); +} + +export class ConatService extends EventEmitter { + private options: Options; + public readonly subject: string; + public readonly name: string; + private sub?; + + constructor(options: Options) { + super(); + this.options = options; + this.name = serviceName(this.options); + this.subject = serviceSubject(options); + this.runService(); + } + + private log = (...args) => { + logger.debug(`service:subject='${this.subject}' -- `, ...args); + }; + + // create and run the service until something goes wrong, when this + // willl return. It does not throw an error. + private runService = async () => { + this.emit("starting"); + this.log("starting service", { + name: this.name, + description: this.options.description, + version: this.options.version, + }); + const cn = this.options.client ?? (await conat()); + const queue = this.options.all ? randomId() : "0"; + // service=true so upon disconnect the socketio backend server + // immediately stops routing traffic to this. + this.sub = await cn.subscribe(this.subject, { queue }); + this.emit("running"); + await this.listen(); + }; + + private listen = async () => { + for await (const mesg of this.sub) { + const request = mesg.data ?? {}; + + // console.logger.debug("handle conat service call", request); + let resp; + if (request == "ping") { + resp = "pong"; + } else { + try { + resp = await this.options.handler(request); + } catch (err) { + resp = { error: `${err}` }; + } + } + try { + await mesg.respond(resp); + } catch (err) { + const data = { error: `${err}` }; + await mesg.respond(data); + } + } + }; + + close = () => { + if (!this.subject) { + return; + } + this.emit("closed"); + this.removeAllListeners(); + this.sub?.stop(); + delete this.sub; + // @ts-ignore + delete this.subject; + // @ts-ignore + delete this.options; + }; +} + +interface ServiceClientOpts { + options: ServiceDescription; + maxWait?: number; + id?: string; +} + +export async function pingConatService({ + options, + maxWait = 3000, +}: ServiceClientOpts): Promise { + const pong = await callConatService({ + ...options, + mesg: "ping", + timeout: Math.max(3000, maxWait), + // set no-retry to avoid infinite loop + noRetry: true, + }); + return [pong]; +} + +// NOTE: anything that has to rely on waitForConatService should +// likely be rewritten differently... +export async function waitForConatService({ + options, + maxWait = 60000, +}: { + options: ServiceDescription; + maxWait?: number; +}) { + let ping: string[] = []; + let pingMaxWait = 250; + await until( + async () => { + pingMaxWait = Math.min(3000, pingMaxWait * 1.4); + try { + ping = await pingConatService({ options, maxWait: pingMaxWait }); + return ping.length > 0; + } catch { + return false; + } + }, + { + start: 1000, + max: 10000, + decay: 1.3, + timeout: maxWait, + }, + ); + return ping; +} diff --git a/src/packages/nats/service/syncfs-client.ts b/src/packages/conat/service/syncfs-client.ts similarity index 100% rename from src/packages/nats/service/syncfs-client.ts rename to src/packages/conat/service/syncfs-client.ts diff --git a/src/packages/nats/service/syncfs-server.ts b/src/packages/conat/service/syncfs-server.ts similarity index 100% rename from src/packages/nats/service/syncfs-server.ts rename to src/packages/conat/service/syncfs-server.ts diff --git a/src/packages/nats/service/terminal.ts b/src/packages/conat/service/terminal.ts similarity index 69% rename from src/packages/nats/service/terminal.ts rename to src/packages/conat/service/terminal.ts index e69c0de0b4..4956e0506b 100644 --- a/src/packages/nats/service/terminal.ts +++ b/src/packages/conat/service/terminal.ts @@ -2,7 +2,13 @@ Service for controlling a terminal served from a project/compute server. */ -import { createServiceClient, createServiceHandler } from "./typed"; +import { + createServiceClient, + createServiceHandler, + type ConatService, +} from "./typed"; + +export type { ConatService }; export const SIZE_TIMEOUT_MS = 45000; @@ -36,29 +42,30 @@ interface TerminalApi { close: (browser_id: string) => Promise; } -export function createTerminalClient({ project_id, path }) { +export function createTerminalClient({ project_id, termPath }) { return createServiceClient({ project_id, - path, - service: "project-api", + path: termPath, + service: "terminal-server", + timeout: 3000, }); } export type TerminalServiceApi = ReturnType; -export async function createTerminalServer({ +export function createTerminalServer({ project_id, - path, + termPath, impl, }: { project_id: string; - path: string; + termPath: string; impl; -}) { - return await createServiceHandler({ +}): ConatService { + return createServiceHandler({ project_id, - path, - service: "project-api", + path: termPath, + service: "terminal-server", description: "Terminal service.", impl, }); @@ -77,27 +84,27 @@ export interface TerminalBrowserApi { size: (opts: { rows: number; cols: number }) => Promise; } -export function createBrowserClient({ project_id, path }) { +export function createBrowserClient({ project_id, termPath }) { return createServiceClient({ project_id, - path, - service: "browser-api", + path: termPath, + service: "terminal-browser", }); } -export async function createBrowserService({ +export function createBrowserService({ project_id, - path, + termPath, impl, }: { project_id: string; - path: string; + termPath: string; impl: TerminalBrowserApi; -}) { - return await createServiceHandler({ +}): ConatService { + return createServiceHandler({ project_id, - path, - service: "browser-api", + path: termPath, + service: "terminal-browser", description: "Browser Terminal service.", all: true, impl, diff --git a/src/packages/nats/service/time.ts b/src/packages/conat/service/time.ts similarity index 95% rename from src/packages/nats/service/time.ts rename to src/packages/conat/service/time.ts index 30a1f7bc40..d5c6c41c65 100644 --- a/src/packages/nats/service/time.ts +++ b/src/packages/conat/service/time.ts @@ -5,7 +5,7 @@ This is a global service that is run by hubs. */ import { createServiceClient, createServiceHandler } from "./typed"; -import { getClient } from "@cocalc/nats/client"; +import { getClient } from "@cocalc/conat/client"; interface TimeApi { // time in ms since epoch, i.e., Date.now() diff --git a/src/packages/nats/service/typed.ts b/src/packages/conat/service/typed.ts similarity index 51% rename from src/packages/nats/service/typed.ts rename to src/packages/conat/service/typed.ts index f8bd8dcd2e..2f43deeaf3 100644 --- a/src/packages/nats/service/typed.ts +++ b/src/packages/conat/service/typed.ts @@ -1,22 +1,20 @@ import { - callNatsService, - createNatsService, - natsServiceInfo, - natsServiceStats, - pingNatsService, - waitForNatsService, + callConatService, + createConatService, + pingConatService, + waitForConatService, + type ConatService, } from "./service"; import type { Options, ServiceCall } from "./service"; +export type { ConatService }; export interface Extra { - info: typeof natsServiceInfo; - stats: typeof natsServiceStats; - ping: typeof pingNatsService; + ping: typeof pingConatService; waitFor: (opts?: { maxWait?: number }) => Promise; } export interface ServiceApi { - nats: Extra; + conat: Extra; } export function createServiceClient(options: Omit) { @@ -27,26 +25,22 @@ export function createServiceClient(options: Omit) { if (typeof prop !== "string") { return undefined; } - if (prop == "nats") { + if (prop == "conat") { return { - info: async (opts: { id?: string; maxWait?: number } = {}) => - await natsServiceInfo({ options, ...opts }), - stats: async (opts: { id?: string; maxWait?: number } = {}) => - await natsServiceStats({ options, ...opts }), ping: async (opts: { id?: string; maxWait?: number } = {}) => - await pingNatsService({ options, ...opts }), + await pingConatService({ options, ...opts }), waitFor: async (opts: { maxWait?: number } = {}) => - await waitForNatsService({ options, ...opts }), + await waitForConatService({ options, ...opts }), }; } return async (...args) => { try { - return await callNatsService({ + return await callConatService({ ...options, mesg: { name: prop, args }, }); } catch (err) { - err.message = `Error calling remote function '${prop}': ${err.message}`; + err.message = `calling remote function '${prop}': ${err.message}`; throw err; } }; @@ -55,11 +49,11 @@ export function createServiceClient(options: Omit) { ) as Api & ServiceApi; } -export async function createServiceHandler({ +export function createServiceHandler({ impl, ...options -}: Omit & { impl: Api }) { - return await createNatsService({ +}: Omit & { impl: Api }): ConatService { + return createConatService({ ...options, handler: async (mesg) => await impl[mesg.name](...mesg.args), }); diff --git a/src/packages/conat/socket/README.md b/src/packages/conat/socket/README.md new file mode 100644 index 0000000000..17b5ed056b --- /dev/null +++ b/src/packages/conat/socket/README.md @@ -0,0 +1,95 @@ +# SOCKETS + +In compute networking, **TCP sockets** are a great idea that's been around since 1974! They are +incredibly useful as an abstraction. To create +a TCP socket you define source and target ports and ip address, and have a client +and server that are on a common network, so the client can connect to +the server. On the other hand, conat's pub/sub model lets you +instead have all clients/servers connect to a common "fabric" +and publish and subscribe using subject patterns and subjects. +This is extremley nice because there's no notion of ip addresses, +and clients and servers do not have to be directly connected to +each other. + +**The TCP protocol for sockets guarantees **in-order, reliable, and +lossless transmission of messages between sender and receiver.** +That same guarantee is thus what we support with our socket abstraction. + +This module provides an emulation of sockets but on top of the +conat pub/sub model. The server and clients agree on a common +*subject* pattern of the form `${subject}.>` that they both +have read/write permissions for. Then the server listens for +new socket connections from clients. Sockets get setup and +the server can write to each one, they can write to the server, +and the server can broadcast to all connected sockets. +There are heartbeats to keep everything alive. When a client +or server properly closes a connection, the other side gets +immediately notified. + +Of course you can also send arbitrary messages over the socket. + +STATES: + +- disconnected \- not actively sending or receiving messages. You can write to the socket and messages will be buffered to be sent when connected. +- connecting \- in the process of connecting +- ready \- actively connected and listening for incoming messages +- closed: _nothing_ further can be done with the socket. + +A socket can be closed by the remote side. + +LOAD BALANCING AND AUTOMATIC FAILOVER: + +We use a *sticky* subscription on the server's side. This means +you can have several distinct socket servers for the same subject, +and connection will get distributed between them, but once a connection +is created, it will persist in the expected way (i.e., the socket +connects with exactly one choice of server). You can dynamically +add and remove servers at any time. You get stateful automatic +load balancing and automatic across all of them. + +HEADERS ARE FULLY SUPPORTED: + +If you just use s.write(data) and s.on('data', (data)=>) then +you get the raw data without headers. However, headers -- arbitrary +JSON separate from the raw (possibly binary) payload -- are supported. +You just have to pass a second argument: + s.write(data, headers) and s.on('data', (data,headers) => ...) + +UNIT TESTS: + +For unit tests, see + + backend/conat/test/socket/conat-socket.test.ts + +WARNING: + +If you create a socket server on with a given subject, then +it will use `${subject}.server.*` and `${subject}.client.*`, so +don't use `${subject}.>` for anything else! + +DEVELOPMENT: + +Start node via + +``` +CONAT_SERVER=http://localhost:3000 node + +// conat socketio server + +s = await require('@cocalc/server/conat/socketio').initConatServer({port:3000}); 0 + +// server side of socket + +conat = await require('@cocalc/backend/conat').conat(); s = conat.socket.listen('conat.io');s.on('connection',(socket)=>{ + console.log("got new connection", socket.id); + socket.on('data',(data) => console.log("got", {data})); + socket.on('request', (mesg)=>{console.log("responding..."); mesg.respondSync('foo')}) +});0 + +// client side of socket + +conat = await require('@cocalc/backend/conat').conat(); c = conat.socket.connect('conat.io');c.on('data',(data) => console.log("got", {data}));0 + +c.write('hi') +``` + diff --git a/src/packages/conat/socket/base.ts b/src/packages/conat/socket/base.ts new file mode 100644 index 0000000000..5c926977fc --- /dev/null +++ b/src/packages/conat/socket/base.ts @@ -0,0 +1,148 @@ +import { EventEmitter } from "events"; +import { + type Client, + type Subscription, + DEFAULT_REQUEST_TIMEOUT, +} from "@cocalc/conat/core/client"; +import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; +import { once } from "@cocalc/util/async-utils"; +import { + type Role, + type State, + DEFAULT_MAX_QUEUE_SIZE, + type ConatSocketOptions, + RECONNECT_DELAY, + DEFAULT_KEEP_ALIVE, + DEFAULT_KEEP_ALIVE_TIMEOUT, +} from "./util"; +import { type ServerSocket } from "./server-socket"; + +export abstract class ConatSocketBase extends EventEmitter { + public readonly desc?: string; + subject: string; + client: Client; + role: Role; + id: string; + subscribe: string; + sockets: { [id: string]: ServerSocket } = {}; + subjects: { + server: string; + client: string; + }; + + sub?: Subscription; + state: State = "disconnected"; + reconnection: boolean; + ended: boolean = false; + maxQueueSize: number; + keepAlive: number; + keepAliveTimeout: number; + + // the following is all for compat with primus's api and has no meaning here. + address = { ip: "" }; + conn: { id: string }; + OPEN = 1; + CLOSE = 0; + readyState: 0; + // end compat + + constructor({ + subject, + client, + role, + id, + reconnection = true, + maxQueueSize = DEFAULT_MAX_QUEUE_SIZE, + keepAlive = DEFAULT_KEEP_ALIVE, + keepAliveTimeout = DEFAULT_KEEP_ALIVE_TIMEOUT, + desc, + }: ConatSocketOptions) { + super(); + this.maxQueueSize = maxQueueSize; + this.reconnection = reconnection; + this.subject = subject; + this.client = client; + this.role = role; + this.id = id; + this.keepAlive = keepAlive; + this.keepAliveTimeout = keepAliveTimeout; + this.desc = desc; + this.conn = { id }; + this.connect(); + this.setMaxListeners(100); + } + + abstract channel(channel: string); + + protected abstract run(); + + abstract end(opts: { timeout?: number }); + + protected abstract initTCP(); + + protected setState = (state: State) => { + this.state = state; + this.emit(state); + }; + + destroy = () => this.close(); + + close() { + if (this.state == "closed") { + return; + } + this.setState("closed"); + this.removeAllListeners(); + + this.sub?.close(); + delete this.sub; + for (const id in this.sockets) { + this.sockets[id].destroy(); + } + this.sockets = {}; + // @ts-ignore + delete this.client; + } + + disconnect = () => { + if (this.state == "closed") { + return; + } + this.setState("disconnected"); + this.sub?.close(); + delete this.sub; + for (const id in this.sockets) { + this.sockets[id].disconnect(); + } + if (this.reconnection) { + setTimeout(() => { + this.connect(); + }, RECONNECT_DELAY); + } + }; + + connect = async () => { + if (this.state != "disconnected") { + // already connected + return; + } + this.setState("connecting"); + try { + await this.run(); + } catch (err) { + // console.log(`WARNING: ${this.role} socket connect error -- ${err}`); + this.disconnect(); + } + }; + + // usually all the timeouts are the same, so this reuseInFlight is very helpful + waitUntilReady = reuseInFlight(async (timeout?: number) => { + if (this.state == "ready") { + return; + } + await once(this, "ready", timeout ?? DEFAULT_REQUEST_TIMEOUT); + if (this.state == "closed") { + throw Error("closed"); + } + }); +} diff --git a/src/packages/conat/socket/client.ts b/src/packages/conat/socket/client.ts new file mode 100644 index 0000000000..14390501e0 --- /dev/null +++ b/src/packages/conat/socket/client.ts @@ -0,0 +1,291 @@ +import { + messageData, + type Subscription, + type Headers, + ConatError, +} from "@cocalc/conat/core/client"; +import { ConatSocketBase } from "./base"; +import { type TCP, createTCP } from "./tcp"; +import { + SOCKET_HEADER_CMD, + DEFAULT_COMMAND_TIMEOUT, + type ConatSocketOptions, +} from "./util"; +import { EventIterator } from "@cocalc/util/event-iterator"; +import { keepAlive, KeepAlive } from "./keepalive"; +import { getLogger } from "@cocalc/conat/client"; +import { until } from "@cocalc/util/async-utils"; + +const logger = getLogger("socket:client"); + +// DO NOT directly instantiate here -- instead, call the +// socket.connect method on ConatClient. + +export class ConatSocketClient extends ConatSocketBase { + queuedWrites: { data: any; headers?: Headers }[] = []; + private tcp?: TCP; + private alive?: KeepAlive; + + constructor(opts: ConatSocketOptions) { + super(opts); + logger.silly("creating a client socket connecting to ", this.subject); + this.initTCP(); + this.on("ready", () => { + for (const mesg of this.queuedWrites) { + this.sendDataToServer(mesg); + } + }); + if (this.tcp == null) { + throw Error("bug"); + } + } + + channel(channel: string) { + return this.client.socket.connect(this.subject + "." + channel, { + desc: `${this.desc ?? ""}.channel('${channel}')`, + maxQueueSize: this.maxQueueSize, + }) as ConatSocketClient; + } + + private initKeepAlive = () => { + this.alive?.close(); + this.alive = keepAlive({ + role: "client", + ping: async () => + await this.request(null, { + headers: { [SOCKET_HEADER_CMD]: "ping" }, + timeout: this.keepAliveTimeout, + }), + disconnect: this.disconnect, + keepAlive: this.keepAlive, + }); + }; + + initTCP() { + if (this.tcp != null) { + throw Error("this.tcp already initialized"); + } + // request = send a socket request mesg to the server side of the socket + // either ack what's received or asking for a resend of missing data. + const request = async (mesg, opts?) => + await this.client.request(`${this.subject}.server.${this.id}`, mesg, { + ...opts, + headers: { ...opts?.headers, [SOCKET_HEADER_CMD]: "socket" }, + }); + + this.tcp = createTCP({ + request, + role: this.role, + reset: this.disconnect, + send: this.sendToServer, + size: this.maxQueueSize, + }); + + this.client.on("disconnected", this.tcp.send.resendLastUntilAcked); + + this.tcp.recv.on("message", (mesg) => { + this.emit("data", mesg.data, mesg.headers); + }); + this.tcp.send.on("drain", () => { + this.emit("drain"); + }); + } + + waitUntilDrain = async () => { + await this.tcp?.send.waitUntilDrain(); + }; + + private sendCommandToServer = async ( + cmd: "close" | "ping" | "connect", + timeout = DEFAULT_COMMAND_TIMEOUT, + ) => { + logger.silly("sendCommandToServer", { cmd, timeout }); + const headers = { + [SOCKET_HEADER_CMD]: cmd, + id: this.id, + }; + const subject = `${this.subject}.server.${this.id}`; + const resp = await this.client.request(subject, null, { + headers, + timeout, + }); + const value = resp.data; + logger.silly("sendCommandToServer: got resp", { cmd, value }); + if (value?.error) { + throw Error(value?.error); + } else { + return value; + } + }; + + protected async run() { + if (this.state == "closed") { + return; + } + // console.log( + // "client socket -- subscribing to ", + // `${this.subject}.client.${this.id}`, + // ); + try { + logger.silly("run: getting subscription"); + this.sub = await this.client.subscribe( + `${this.subject}.client.${this.id}`, + ); + // @ts-ignore + if (this.state == "closed") { + return; + } + let resp: any = undefined; + await until( + async () => { + if (this.state == "closed") { + logger.silly("closed -- giving up on connecting"); + return true; + } + try { + logger.silly("sending connect command to server"); + resp = await this.sendCommandToServer("connect"); + this.alive?.recv(); + return true; + } catch (err) { + logger.silly("failed to connect", err); + } + return false; + }, + { start: 500, decay: 1.3, max: 10000 }, + ); + + if (resp != "connected") { + throw Error("failed to connect"); + } + this.setState("ready"); + this.initKeepAlive(); + for await (const mesg of this.sub) { + this.alive?.recv(); + const cmd = mesg.headers?.[SOCKET_HEADER_CMD]; + if (cmd) { + logger.silly("client got cmd", cmd); + } + if (cmd == "socket") { + this.tcp?.send.handleRequest(mesg); + } else if (cmd == "close") { + this.close(); + return; + } else if (cmd == "ping") { + // logger.silly("responding to ping from server", this.id); + mesg.respond(null); + } else if (mesg.isRequest()) { + // logger.silly("client got request"); + this.emit("request", mesg); + } else { + // logger.silly("client got data"); //, { data: mesg.data }); + this.tcp?.recv.process(mesg); + } + } + } catch (err) { + logger.silly("socket connect failed", err); + this.disconnect(); + } + } + + private sendDataToServer = (mesg) => { + const subject = `${this.subject}.server.${this.id}`; + this.client.publishSync(subject, null, { + raw: mesg.raw, + headers: mesg.headers, + }); + }; + + private sendToServer = (mesg) => { + if (this.state != "ready") { + this.queuedWrites.push(mesg); + while (this.queuedWrites.length > this.maxQueueSize) { + this.queuedWrites.shift(); + } + return; + } + // @ts-ignore + if (this.state == "closed") { + throw Error("closed"); + } + if (this.role == "server") { + throw Error("sendToServer is only for use by the client"); + } else { + // we are the client, so write to server + this.sendDataToServer(mesg); + } + }; + + request = async (data, options?) => { + await this.waitUntilReady(options?.timeout); + const subject = `${this.subject}.server.${this.id}`; + if (this.state == "closed") { + throw Error("closed"); + } + // console.log("sending request from client ", { subject, data, options }); + return await this.client.request(subject, data, options); + }; + + requestMany = async (data, options?): Promise => { + await this.waitUntilReady(options?.timeout); + const subject = `${this.subject}.server.${this.id}`; + return await this.client.requestMany(subject, data, options); + }; + + async end({ timeout = 3000 }: { timeout?: number } = {}) { + if (this.state == "closed") { + return; + } + this.reconnection = false; + this.ended = true; + // tell server we're done + try { + await this.sendCommandToServer("close", timeout); + } catch {} + this.close(); + } + + close() { + if (this.state == "closed") { + return; + } + this.sub?.close(); + if (this.tcp != null) { + this.client.removeListener( + "disconnected", + this.tcp.send.resendLastUntilAcked, + ); + } + this.queuedWrites = []; + // tell server we're gone (but don't wait) + (async () => { + try { + await this.sendCommandToServer("close"); + } catch {} + })(); + if (this.tcp != null) { + this.tcp.send.close(); + this.tcp.recv.close(); + // @ts-ignore + delete this.tcp; + } + this.alive?.close(); + delete this.alive; + super.close(); + } + + // writes will raise an exception if: (1) the socket is closed code='EPIPE', or (2) + // you hit maxQueueSize un-ACK'd messages, code='ENOBUFS' + write = (data, { headers }: { headers?: Headers } = {}): void => { + // @ts-ignore + if (this.state == "closed") { + throw new ConatError("closed", { code: "EPIPE" }); + } + const mesg = messageData(data, { headers }); + this.tcp?.send.process(mesg); + }; + + iter = () => { + return new EventIterator<[any, Headers]>(this, "data"); + }; +} diff --git a/src/packages/conat/socket/index.ts b/src/packages/conat/socket/index.ts new file mode 100644 index 0000000000..488d2ffbfd --- /dev/null +++ b/src/packages/conat/socket/index.ts @@ -0,0 +1,8 @@ +/* + +*/ + +export { ServerSocket } from "./server-socket"; +export { ConatSocketClient } from "./client"; +export { ConatSocketServer } from "./server"; +export type { SocketConfiguration } from "./util"; diff --git a/src/packages/conat/socket/keepalive.ts b/src/packages/conat/socket/keepalive.ts new file mode 100644 index 0000000000..3ba6e8ed06 --- /dev/null +++ b/src/packages/conat/socket/keepalive.ts @@ -0,0 +1,63 @@ +import { delay } from "awaiting"; +import { getLogger } from "@cocalc/conat/client"; +import { type Role } from "./util"; + +const logger = getLogger("socket:keepalive"); + +export function keepAlive(opts: { + role: Role; + ping: () => Promise; + disconnect: () => void; + keepAlive: number; +}) { + return new KeepAlive(opts.ping, opts.disconnect, opts.keepAlive, opts.role); +} + +export class KeepAlive { + private last: number = Date.now(); + private state: "ready" | "closed" = "ready"; + + constructor( + private ping: () => Promise, + private disconnect: () => void, + private keepAlive: number, + private role: Role, + ) { + this.run(); + } + + private run = async () => { + while (this.state == "ready") { + try { + logger.silly(this.role, "keepalive -- sending ping"); + await this.ping?.(); + } catch (err) { + logger.silly(this.role, "keepalive -- ping failed -- disconnecting"); + this.disconnect?.(); + this.close(); + return; + } + this.last = Date.now(); + if (this.state == ("closed" as any)) { + return; + } + await delay(this.keepAlive - (Date.now() - this.last)); + } + }; + + // call this when any data is received, which defers having to waste resources on + // sending a ping + recv = () => { + this.last = Date.now(); + }; + + close = () => { + this.state = "closed"; + // @ts-ignore + delete this.last; + // @ts-ignore + delete this.ping; + // @ts-ignore + delete this.disconnect; + }; +} diff --git a/src/packages/conat/socket/server-socket.ts b/src/packages/conat/socket/server-socket.ts new file mode 100644 index 0000000000..b6d40e27d3 --- /dev/null +++ b/src/packages/conat/socket/server-socket.ts @@ -0,0 +1,235 @@ +import { EventEmitter } from "events"; +import { + type Headers, + DEFAULT_REQUEST_TIMEOUT, + type Message, + messageData, + ConatError, +} from "@cocalc/conat/core/client"; +import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; +import { once } from "@cocalc/util/async-utils"; +import { SOCKET_HEADER_CMD, type State, clientSubject } from "./util"; +import { type TCP, createTCP } from "./tcp"; +import { type ConatSocketServer } from "./server"; +import { keepAlive, KeepAlive } from "./keepalive"; +import { getLogger } from "@cocalc/conat/client"; + +const logger = getLogger("socket:server-socket"); + +// One specific socket from the point of view of a server. +export class ServerSocket extends EventEmitter { + private conatSocket: ConatSocketServer; + public readonly id: string; + public lastPing = Date.now(); + + private queuedWrites: { data: any; headers?: Headers }[] = []; + private clientSubject: string; + + public state: State = "ready"; + // the non-pattern subject the client connected to + public readonly subject: string; + + // this is just for compat with conatSocket api: + public readonly address = { ip: "" }; + // conn is just for compatibility with primus/socketio (?). + public readonly conn: { id: string }; + + public tcp?: TCP; + private alive?: KeepAlive; + + constructor({ conatSocket, id, subject }) { + super(); + this.subject = subject; + this.conatSocket = conatSocket; + this.clientSubject = clientSubject(subject); + this.id = id; + this.conn = { id }; + this.initTCP(); + if (this.tcp == null) { + throw Error("bug"); + } + this.initKeepAlive(); + } + + private initKeepAlive = () => { + this.alive?.close(); + this.alive = keepAlive({ + role: "server", + ping: async () => { + await this.request(null, { + headers: { [SOCKET_HEADER_CMD]: "ping" }, + timeout: this.conatSocket.keepAliveTimeout, + }); + }, + disconnect: this.close, + keepAlive: this.conatSocket.keepAlive, + }); + }; + + initTCP() { + if (this.tcp != null) { + throw Error("this.tcp already initialized"); + } + const request = async (mesg, opts?) => + await this.conatSocket.client.request(this.clientSubject, mesg, { + ...opts, + headers: { ...opts?.headers, [SOCKET_HEADER_CMD]: "socket" }, + }); + this.tcp = createTCP({ + request, + role: this.conatSocket.role, + reset: this.close, + send: this.send, + size: this.conatSocket.maxQueueSize, + }); + this.conatSocket.client.on( + "disconnected", + this.tcp.send.resendLastUntilAcked, + ); + + this.tcp.recv.on("message", (mesg) => { + // console.log("tcp recv emitted message", mesg.data); + this.emit("data", mesg.data, mesg.headers); + }); + this.tcp.send.on("drain", () => { + this.emit("drain"); + }); + } + + disconnect = () => { + this.setState("disconnected"); + if (this.conatSocket.state == "ready") { + this.setState("ready"); + } else { + this.conatSocket.once("ready", this.onServerSocketReady); + } + }; + + private onServerSocketReady = () => { + if (this.state != "closed") { + this.setState("ready"); + } + }; + + private setState = (state: State) => { + this.state = state; + if (state == "ready") { + for (const mesg of this.queuedWrites) { + this.sendDataToClient(mesg); + } + this.queuedWrites = []; + } + this.emit(state); + }; + + end = async ({ timeout = 3000 }: { timeout?: number } = {}) => { + if (this.state == "closed") { + return; + } + try { + await this.conatSocket.client.publish(this.clientSubject, null, { + headers: { [SOCKET_HEADER_CMD]: "close" }, + timeout, + }); + } catch (err) { + console.log(`WARNING: error closing socket - ${err}`); + } + this.close(); + }; + + destroy = () => this.close(); + + close = () => { + if (this.state == "closed") { + return; + } + this.conatSocket.removeListener("ready", this.onServerSocketReady); + this.conatSocket.client.publishSync(this.clientSubject, null, { + headers: { [SOCKET_HEADER_CMD]: "close" }, + }); + + if (this.tcp != null) { + this.conatSocket.client.removeListener( + "disconnected", + this.tcp.send.resendLastUntilAcked, + ); + this.tcp.send.close(); + this.tcp.recv.close(); + // @ts-ignore + delete this.tcp; + } + + this.alive?.close(); + delete this.alive; + + this.queuedWrites = []; + this.setState("closed"); + this.removeAllListeners(); + delete this.conatSocket.sockets[this.id]; + // @ts-ignore + delete this.conatSocket; + }; + + receiveDataFromClient = (mesg) => { + this.alive?.recv(); + this.tcp?.recv.process(mesg); + }; + + private sendDataToClient = (mesg) => { + this.conatSocket.client.publishSync(this.clientSubject, null, { + raw: mesg.raw, + headers: mesg.headers, + }); + }; + + private send = (mesg: Message) => { + if (this.state != "ready") { + this.queuedWrites.push(mesg); + while (this.queuedWrites.length > this.conatSocket.maxQueueSize) { + this.queuedWrites.shift(); + } + return; + } + // @ts-ignore + if (this.state == "closed") { + return; + } + this.sendDataToClient(mesg); + return true; + }; + + // writes will raise an exception if: (1) the socket is closed, or (2) + // you hit maxQueueSize un-ACK'd messages. + write = (data, { headers }: { headers?: Headers } = {}) => { + if (this.state == "closed") { + throw new ConatError("closed", { code: "EPIPE" }); + } + const mesg = messageData(data, { headers }); + this.tcp?.send.process(mesg); + }; + + // use request reply where the client responds + request = async (data, options?) => { + this.waitUntilReady(options?.timeout); + logger.silly("server sending request to ", this.clientSubject); + return await this.conatSocket.client.request( + this.clientSubject, + data, + options, + ); + }; + + private waitUntilReady = reuseInFlight(async (timeout?: number) => { + if (this.state == "ready") { + return; + } + await once(this, "ready", timeout ?? DEFAULT_REQUEST_TIMEOUT); + if (this.state == "closed") { + throw Error("closed"); + } + }); + + waitUntilDrain = async () => { + await this.tcp?.send.waitUntilDrain(); + }; +} diff --git a/src/packages/conat/socket/server.ts b/src/packages/conat/socket/server.ts new file mode 100644 index 0000000000..82a55f0840 --- /dev/null +++ b/src/packages/conat/socket/server.ts @@ -0,0 +1,194 @@ +import { ConatSocketBase } from "./base"; +import { + PING_PONG_INTERVAL, + type Command, + SOCKET_HEADER_CMD, + clientSubject, +} from "./util"; +import { ServerSocket } from "./server-socket"; +import { delay } from "awaiting"; +import { type Headers } from "@cocalc/conat/core/client"; +import { getLogger } from "@cocalc/conat/client"; + +const logger = getLogger("socket:server"); + +// DO NOT directly instantiate here -- instead, call the +// socket.listen method on ConatClient. + +export class ConatSocketServer extends ConatSocketBase { + initTCP() {} + + channel(channel: string) { + return this.client.socket.listen(this.subject + "." + channel, { + desc: `${this.desc ?? ""}.channel('${channel}')`, + }) as ConatSocketServer; + } + + forEach = (f: (socket: ServerSocket, id: string) => void) => { + for (const id in this.sockets) { + f(this.sockets[id], id); + } + }; + + protected async run() { + this.deleteDeadSockets(); + const sub = await this.client.subscribe(`${this.subject}.server.*`, { + sticky: true, + }); + if (this.state == "closed") { + sub.close(); + return; + } + this.sub = sub; + this.setState("ready"); + for await (const mesg of this.sub) { + // console.log("got mesg", mesg.data, mesg.headers); + if (this.state == ("closed" as any)) { + return; + } + const cmd = mesg.headers?.[SOCKET_HEADER_CMD]; + const id = mesg.subject.split(".").slice(-1)[0]; + let socket = this.sockets[id]; + + if (socket === undefined) { + if (cmd == "close") { + // already closed + continue; + } + // not connected yet -- anything except a connect message is ignored. + if (cmd != "connect") { + logger.debug( + "ignoring data from not-connected socket -- telling it to close", + { id, cmd }, + ); + this.client.publishSync(clientSubject(mesg.subject), null, { + headers: { [SOCKET_HEADER_CMD]: "close" }, + }); + continue; + } + // new connection + socket = new ServerSocket({ + conatSocket: this, + id, + subject: mesg.subject, + }); + this.sockets[id] = socket; + this.emit("connection", socket); + } + + if (cmd !== undefined) { + // note: test this first since it is also a request + // a special internal control command + this.handleCommandFromClient({ socket, cmd: cmd as Command, mesg }); + } else if (mesg.isRequest()) { + // a request to support the socket.on('request', (mesg) => ...) protocol: + socket.emit("request", mesg); + } else { + socket.receiveDataFromClient(mesg); + } + } + } + + private async deleteDeadSockets() { + while (this.state != "closed") { + for (const id in this.sockets) { + const socket = this.sockets[id]; + if (Date.now() - socket.lastPing > PING_PONG_INTERVAL * 2.5) { + socket.destroy(); + } + } + await delay(PING_PONG_INTERVAL); + } + } + + request = async (data, options?) => { + await this.waitUntilReady(options?.timeout); + + // we call all connected sockets in parallel, + // then return array of responses. + // Unless race is set, then we return first result + const v: any[] = []; + for (const id in this.sockets) { + const f = async () => { + if (this.state == "closed") { + throw Error("closed"); + } + try { + return await this.sockets[id].request(data, options); + } catch (err) { + return err; + } + }; + v.push(f()); + } + if (options?.race) { + return await Promise.race(v); + } else { + return await Promise.all(v); + } + }; + + write = (data, { headers }: { headers?: Headers } = {}): void => { + // @ts-ignore + if (this.state == "closed") { + throw Error("closed"); + } + // write to all the sockets that are connected. + for (const id in this.sockets) { + this.sockets[id].write(data, headers); + } + }; + + handleCommandFromClient = ({ + socket, + cmd, + mesg, + }: { + socket: ServerSocket; + cmd: Command; + mesg; + }) => { + socket.lastPing = Date.now(); + if (cmd == "socket") { + socket.tcp?.send.handleRequest(mesg); + } else if (cmd == "ping") { + if (socket.state == "ready") { + // ONLY respond to ping for a server socket if that socket is + // actually ready! ping's are meant to check whether the server + // socket views itself as connected right now. If not, connected, + // ping should timeout + logger.silly("responding to ping from client", socket.id); + mesg.respondSync(null); + } + } else if (cmd == "close") { + const id = socket.id; + socket.close(); + delete this.sockets[id]; + mesg.respondSync("closed"); + } else if (cmd == "connect") { + mesg.respondSync("connected"); + } else { + mesg.respondSync({ error: `unknown command - '${cmd}'` }); + } + }; + + async end({ timeout = 3000 }: { timeout?: number } = {}) { + if (this.state == "closed") { + return; + } + this.reconnection = false; + this.ended = true; + // tell all clients to end + const end = async (id) => { + const socket = this.sockets[id]; + delete this.sockets[id]; + try { + await socket.end({ timeout }); + } catch (err) { + console.log("WARNING: error ending socket -- ${err}"); + } + }; + await Promise.all(Object.keys(this.sockets).map(end)); + this.close(); + } +} diff --git a/src/packages/conat/socket/tcp.ts b/src/packages/conat/socket/tcp.ts new file mode 100644 index 0000000000..7f0e2b4994 --- /dev/null +++ b/src/packages/conat/socket/tcp.ts @@ -0,0 +1,291 @@ +/* +This is an implementation of the core idea of TCP, i.e., +it is a "transmission control protocol", which ensures +in order exactly once message delivery. +*/ + +import { SOCKET_HEADER_SEQ, type Role } from "./util"; +import { EventEmitter } from "events"; +import { + type Message, + messageData, + type MessageData, + ConatError, +} from "@cocalc/conat/core/client"; +import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; +import { once, until } from "@cocalc/util/async-utils"; + +const DEFAULT_TIMEOUT = 2 * 60 * 1000; + +export interface TCP { + send: Sender; + recv: Receiver; +} + +export function createTCP({ request, send, reset, role, size }): TCP { + return { + send: new Sender(send, role, size), + recv: new Receiver(request, reset, role), + }; +} + +export class Receiver extends EventEmitter { + private incoming?: { [id: number]: MessageData } = {}; + private seq?: { + // next = seq of the next message we should emit + next: number; + // emitted = seq of the last message we actually did emit + emitted: number; + // reported = seq of last message we reported received to caller + reported: number; + // largest = largest seq of any message we have received + largest: number; + } = { next: 1, emitted: 0, reported: 0, largest: 0 }; + + constructor( + private request, + private reset, + public readonly role: Role, + ) { + super(); + } + + close = () => { + this.removeAllListeners(); + delete this.incoming; + delete this.seq; + }; + + process = (mesg: MessageData) => { + if (this.seq === undefined || this.incoming === undefined) return; + const seq = mesg.headers?.[SOCKET_HEADER_SEQ]; + // console.log(this.role, "recv", { data: mesg.data, seq }); + if (typeof seq != "number" || seq < 1) { + console.log( + `WARNING: ${this.role} discarding message -- seq must be a positive integer`, + { seq, mesg: mesg.data, headers: mesg.headers }, + ); + return; + } + this.seq.largest = Math.max(seq, this.seq.largest); + // console.log("process", { seq, next: this.seq.next }); + if (seq == this.seq.next) { + this.emitMessage(mesg, seq); + } else if (seq > this.seq.next) { + // in the future -- save until we get this.seq.next: + this.incoming[seq] = mesg; + // console.log("doing fetchMissing because: ", { seq, next: this.seq.next }); + this.fetchMissing(); + } + }; + + private emitMessage = (mesg, seq) => { + if (this.seq === undefined) return; + if (seq != this.seq.next) { + throw Error("message sequence is wrong"); + } + this.seq.next = seq + 1; + this.seq.emitted = seq; + delete mesg.headers?.[SOCKET_HEADER_SEQ]; + // console.log("emitMessage", mesg.data, { + // seq, + // next: this.seq.next, + // emitted: this.seq.emitted, + // }); + // console.log(this.role, "tcp recv", seq, mesg.data); + this.emit("message", mesg); + this.reportReceived(); + }; + + private fetchMissing = reuseInFlight(async () => { + if (this.seq === undefined || this.incoming === undefined) return; + const missing: number[] = []; + for (let seq = this.seq.next; seq <= this.seq.largest; seq++) { + if (this.incoming[seq] === undefined) { + missing.push(seq); + } + } + if (missing.length == 0) { + return; + } + missing.sort(); + let resp; + try { + resp = await this.request({ socket: { missing } }); + } catch (err) { + // 503 happens when the other side is temporarily not available + // if (err.code != 503) { + // console.log("WARNING: error requesting missing messages", missing, err); + // } + return; + } + if (this.seq == null) { + return; + } + if (resp.headers?.error) { + // missing data doesn't exist -- must reset + this.reset(); + return; + } + // console.log("got missing", resp.data); + for (const x of resp.data) { + this.process(messageData(null, x)); + } + this.emitIncoming(); + }); + + private emitIncoming = () => { + if (this.seq === undefined || this.incoming === undefined) return; + // also emit any incoming that comes next + let seq = this.seq.next; + while (this.incoming[seq] != null && this.seq != null) { + const mesg = this.incoming[seq]; + delete this.incoming[seq]; + this.emitMessage(mesg, seq); + seq += 1; + } + this.reportReceived(); + }; + + private reportReceived = async () => { + if (this.seq === undefined) return; + if (this.seq.reported >= this.seq.emitted) { + // nothing to report + return; + } + const x = { socket: { emitted: this.seq.emitted } }; + try { + await this.request(x); + if (this.seq == null) { + return; + } + this.seq.reported = x.socket.emitted; + } catch { + // When things are broken this should throw, and the natura of tcp is that + // things should sometimes be broken. + } + }; +} + +export class Sender extends EventEmitter { + private outgoing: { [id: number]: Message } = {}; + private seq = 0; + timeout = DEFAULT_TIMEOUT; + private unsent: number = 0; + + constructor( + private send: (mesg: Message) => void, + public readonly role: Role, + private size: number, + ) { + super(); + } + + close = () => { + this.removeAllListeners(); + // @ts-ignore + delete this.outgoing; + // @ts-ignore + delete this.seq; + }; + + process = (mesg) => { + if (this.unsent >= this.size) { + throw new ConatError( + `WRITE FAILED: socket buffer size ${this.size} exceeded`, + { code: "ENOBUFS" }, + ); + } + this.seq += 1; + // console.log("Sender.process", mesg.data, this.seq); + this.outgoing[this.seq] = mesg; + this.unsent++; + mesg.headers = { ...mesg.headers, [SOCKET_HEADER_SEQ]: this.seq }; + // console.log(this.role, "send", { data: mesg.data, seq: this.seq }); + this.send(mesg); + }; + + private lastAcked = (): boolean => { + return this.seq == 0 || this.outgoing[this.seq] === undefined; + }; + + // if socket is suspicious that the most recently sent message may + // have been dropped, they call this. If indeed it was not acknowledged, + // the last message will get sent again, which also will trigger the + // other side of the socket to fetch anything else that it did not receive. + private resendLast = () => { + if (this.lastAcked()) { + // console.log("resendLast -- nothing to do"); + // no-op + } + // console.log("resendLast -- resending"); + this.send(this.outgoing[this.seq]); + }; + + // this gets tested in backend/conat/test/socket/restarts.test.ts + resendLastUntilAcked = reuseInFlight(async () => { + try { + await until( + () => { + if (this.outgoing === undefined || this.lastAcked()) { + // done -- condition satisfied + return true; + } + this.resendLast(); + return false; + }, + { start: 500, max: 15000, decay: 1.3, timeout: this.timeout }, + ); + } catch (_err) { + // it will throw if it hits the timeout -- silently ignore, since + // there's no guarantee resendLastUntilAcked actually succeeds + } + }); + + handleRequest = (mesg) => { + if (mesg.data?.socket == null || this.seq == null) { + return; + } + const { emitted, missing } = mesg.data.socket; + if (emitted != null) { + for (const id in this.outgoing) { + if (parseInt(id) <= emitted) { + delete this.outgoing[id]; + this.unsent--; + if (this.unsent == 0) { + this.emit("drain"); + } + } + } + mesg.respondSync({ emitted }); + } else if (missing != null) { + const v: Message[] = []; + for (const id of missing) { + const x = this.outgoing[id]; + if (x == null) { + // the data does not exist on this client. This should only happen, e.g., + // on automatic failover with the sticky load balancer... ? + mesg.respondSync(null, { headers: { error: "nodata" } }); + return; + } + v.push(x); + } + //console.log("sending missing", v); + mesg.respondSync(v); + } + }; + + waitUntilDrain = reuseInFlight(async () => { + if (this.unsent == 0) { + return; + } + try { + await once(this, "drain"); + } catch (err) { + if (this.outgoing == null) { + return; + } + throw err; + } + }); +} diff --git a/src/packages/conat/socket/util.ts b/src/packages/conat/socket/util.ts new file mode 100644 index 0000000000..b078da2134 --- /dev/null +++ b/src/packages/conat/socket/util.ts @@ -0,0 +1,62 @@ +export const SOCKET_HEADER_CMD = "CN-SocketCmd"; +export const SOCKET_HEADER_SEQ = "CN-SocketSeq"; + +export type State = "disconnected" | "connecting" | "ready" | "closed"; + +export type Role = "client" | "server"; + +// client pings server this frequently and disconnects if +// doesn't get a pong back. Server disconnects client if +// it doesn't get a ping as well. This is NOT the primary +// keep alive/disconnect mechanism -- it's just a backup. +// Primarily we watch the connect/disconnect events from +// socketio and use those to manage things. This ping +// is entirely a "just in case" backup if some event +// were missed (e.g., a kill -9'd process...) +export const PING_PONG_INTERVAL = 60000; + +// We queue up unsent writes, but only up to a point (to not have a huge memory issue). +// Any write beyond this size result in an exception. +// NOTE: in nodejs the default for exactly this is "infinite=use up all RAM", so +// maybe we should make this even larger (?). +// Also note that this is just the *number* of messages, and a message can have +// any size. +export const DEFAULT_MAX_QUEUE_SIZE = 1000; + +export const DEFAULT_COMMAND_TIMEOUT = 3000; + +export const DEFAULT_KEEP_ALIVE = 90000; +export const DEFAULT_KEEP_ALIVE_TIMEOUT = 15000; + +export type Command = "connect" | "close" | "ping" | "socket"; + +import { type Client } from "@cocalc/conat/core/client"; + +export interface SocketConfiguration { + maxQueueSize?: number; + // (Default: true) Whether reconnection is enabled or not. + // If set to false, you need to manually reconnect: + reconnection?: boolean; + // ping other end of the socket if no data is received for keepAlive ms; + // if other side doesn't respond within keepAliveTimeout, then the + // connection switches to the 'disconnected' state. + keepAlive?: number; // default: DEFAULT_KEEP_ALIVE + keepAliveTimeout?: number; // default: DEFAULT_KEEP_ALIVE_TIMEOUT} + // desc = optional, purely for admin/user + desc?: string; +} + +export interface ConatSocketOptions extends SocketConfiguration { + subject: string; + client: Client; + role: Role; + id: string; +} + +export const RECONNECT_DELAY = 500; + +export function clientSubject(subject: string) { + const segments = subject.split("."); + segments[segments.length - 2] = "client"; + return segments.join("."); +} diff --git a/src/packages/conat/sync/akv.ts b/src/packages/conat/sync/akv.ts new file mode 100644 index 0000000000..80354196db --- /dev/null +++ b/src/packages/conat/sync/akv.ts @@ -0,0 +1,158 @@ +/* +Asynchronous Memory-Efficient Access to Key:Value Store + +This provides access to the same data as dkv, except it doesn't download any +data to the client until you actually call get. The calls to get and +set are thus async. + +There is no need to close this because it is stateless. + +[ ] TODO: efficiently get or set many values at once in a single call. This will be +very useful, e.g., for jupyter notebook timetravel browsing. + +DEVELOPMENT: + +~/cocalc/src/packages/backend$ node + +a = await require("@cocalc/backend/conat/sync").dkv({name:'test'}); a.set('x',5) + + +b = require("@cocalc/backend/conat/sync").akv({name:'test'}) +await b.set('x',10) + +a.get('x') + +await b.get('x') + +*/ + +import { + type StorageOptions, + type PersistStreamClient, + stream, +} from "@cocalc/conat/persist/client"; +import { type DKVOptions } from "./dkv"; +import { + type Headers, + messageData, + type Message, +} from "@cocalc/conat/core/client"; +import { storagePath, type User, COCALC_TOMBSTONE_HEADER } from "./core-stream"; +import { connect } from "@cocalc/conat/core/client"; + +export class AKV { + private storage: StorageOptions; + private user: User; + private stream: PersistStreamClient; + + constructor(options: DKVOptions) { + this.user = { + account_id: options.account_id, + project_id: options.project_id, + }; + this.storage = { path: storagePath(options) }; + const client = options.client ?? connect(); + this.stream = stream({ + client, + user: this.user, + storage: this.storage, + }); + } + + close = () => { + this.stream.close(); + }; + + getMessage = async ( + key: string, + { timeout }: { timeout?: number } = {}, + ): Promise | undefined> => { + const mesg = await this.stream.get({ key, timeout }); + if (mesg?.headers?.[COCALC_TOMBSTONE_HEADER]) { + return undefined; + } + return mesg; + }; + + // // Just get one value asynchronously, rather than the entire dkv. + // // If the timeout option is given and the value of key is not set, + // // will wait until that many ms to get the key. + get = async ( + key: string, + opts?: { timeout?: number }, + ): Promise => { + return (await this.getMessage(key, opts))?.data; + }; + + headers = async ( + key: string, + opts?: { timeout?: number }, + ): Promise => { + return (await this.getMessage(key, opts))?.headers; + }; + + time = async ( + key: string, + opts?: { timeout?: number }, + ): Promise => { + const time = (await this.getMessage(key, opts))?.headers?.time; + return time !== undefined ? new Date(time as number) : undefined; + }; + + delete = async (key: string, opts?: { timeout?: number }): Promise => { + await this.set(key, null as any, { + ...opts, + headers: { [COCALC_TOMBSTONE_HEADER]: true }, + }); + }; + + seq = async ( + key: string, + opts?: { timeout?: number }, + ): Promise => { + return (await this.getMessage(key, opts))?.headers?.seq as + | number + | undefined; + }; + + set = async ( + key: string, + value: T, + options?: { + headers?: Headers; + previousSeq?: number; + timeout?: number; + ttl?: number; + msgID?: string; + }, + ): Promise<{ seq: number; time: number }> => { + const { headers, ...options0 } = options ?? {}; + return await this.stream.set({ + key, + messageData: messageData(value, { headers }), + ...options0, + }); + }; + + keys = async ({ timeout }: { timeout?: number } = {}): Promise => { + return await this.stream.keys({ + timeout, + }); + }; + + sqlite = async ( + statement: string, + params?: any[], + { timeout }: { timeout?: number } = {}, + ): Promise => { + return await this.stream.sqlite({ + timeout, + statement, + params, + }); + }; +} + +export function akv(opts: DKVOptions) { + return new AKV(opts); +} diff --git a/src/packages/conat/sync/astream.ts b/src/packages/conat/sync/astream.ts new file mode 100644 index 0000000000..2e92de8741 --- /dev/null +++ b/src/packages/conat/sync/astream.ts @@ -0,0 +1,214 @@ +/* +Asynchronous Memory Efficient Access to Core Stream. + +This provides access to the same data as dstream, except it doesn't download any +data to the client until you actually call get. The calls to get and +set are thus async. + +There is no need to close this because it is stateless. + +[ ] TODO: efficiently get or set many values at once in a single call. This will be +very useful, e.g., for jupyter notebook timetravel browsing. + +DEVELOPMENT: + +~/cocalc/src/packages/backend$ node + +a = await require("@cocalc/backend/conat/sync").dstream({name:'test'}) + + +b = require("@cocalc/backend/conat/sync").astream({name:'test'}) +const {seq} = await b.push('x') + +a.get() // ['x'] + +await b.get(seq) // 'x' + +*/ + +import { + type StorageOptions, + type PersistStreamClient, + stream, +} from "@cocalc/conat/persist/client"; +import { type DStreamOptions } from "./dstream"; +import { + type Headers, + messageData, + type Client, + Message, + decode, +} from "@cocalc/conat/core/client"; +import { storagePath, type User } from "./core-stream"; +import { connect } from "@cocalc/conat/core/client"; +import { type Configuration } from "@cocalc/conat/persist/storage"; + +export class AStream { + private storage: StorageOptions; + private user: User; + private stream: PersistStreamClient; + private client: Client; + + constructor(options: DStreamOptions) { + this.user = { + account_id: options.account_id, + project_id: options.project_id, + }; + this.storage = { path: storagePath(options) }; + this.client = options.client ?? connect(); + this.stream = stream({ + client: this.client, + user: this.user, + storage: this.storage, + }); + } + + close = () => { + this.stream.close(); + }; + + getMessage = async ( + seq_or_key: number | string, + { timeout }: { timeout?: number } = {}, + ): Promise | undefined> => { + return await this.stream.get({ + ...opt(seq_or_key), + timeout, + }); + }; + + get = async ( + seq_or_key: number | string, + opts?: { timeout?: number }, + ): Promise => { + return (await this.getMessage(seq_or_key, opts))?.data; + }; + + headers = async ( + seq_or_key: number | string, + opts?: { timeout?: number }, + ): Promise => { + return (await this.getMessage(seq_or_key, opts))?.headers; + }; + + // this is an async iterator so you can iterate over the + // data without having to have it all in RAM at once. + // Of course, you can put it all in a single list if you want. + async *getAll(opts): AsyncGenerator< + { + mesg: T; + headers?: Headers; + seq: number; + time: number; + key?: string; + }, + void, + unknown + > { + for await (const messages of this.stream.getAll(opts)) { + for (const { seq, time, key, encoding, raw, headers } of messages) { + const mesg = decode({ encoding, data: raw }); + yield { mesg, headers, seq, time, key }; + } + } + } + + async *changefeed(): AsyncGenerator< + | { + op: "set"; + mesg: T; + headers?: Headers; + seq: number; + time: number; + key?: string; + } + | { op: "delete"; seqs: number[] }, + void, + unknown + > { + const cf = await this.stream.changefeed(); + for await (const { updates } of cf) { + for (const event of updates) { + if (event.op == "delete") { + yield event; + } else { + const { seq, time, key, encoding, raw, headers } = event; + const mesg = decode({ encoding, data: raw }); + yield { op: "set", mesg, headers, seq, time, key }; + } + } + } + } + + delete = async (opts: { + timeout?: number; + seq?: number; + last_seq?: number; + all?: boolean; + }): Promise<{ seqs: number[] }> => { + return await this.stream.delete(opts); + }; + + publish = async ( + value: T, + options?: { + headers?: Headers; + previousSeq?: number; + timeout?: number; + key?: string; + ttl?: number; + msgID?: string; + }, + ): Promise<{ seq: number; time: number }> => { + const { headers, ...options0 } = options ?? {}; + return await this.stream.set({ + messageData: messageData(value, { headers }), + ...options0, + }); + }; + + push = async ( + ...args: T[] + ): Promise<({ seq: number; time: number } | { error: string })[]> => { + // [ ] TODO: should break this up into chunks with a limit on size. + const ops = args.map((mesg) => { + return { messageData: messageData(mesg) }; + }); + return await this.stream.setMany(ops); + }; + + config = async ( + config: Partial = {}, + ): Promise => { + if (this.storage == null) { + throw Error("bug -- storage must be set"); + } + return await this.stream.config({ config }); + }; + + sqlite = async ( + statement: string, + params?: any[], + { timeout }: { timeout?: number } = {}, + ): Promise => { + return await this.stream.sqlite({ + timeout, + statement, + params, + }); + }; +} + +export function astream(opts: DStreamOptions) { + return new AStream(opts); +} + +function opt(seq_or_key: number | string): { seq: number } | { key: string } { + const t = typeof seq_or_key; + if (t == "number") { + return { seq: seq_or_key as number }; + } else if (t == "string") { + return { key: seq_or_key as string }; + } + throw Error(`arg must be number or string`); +} diff --git a/src/packages/conat/sync/core-stream.ts b/src/packages/conat/sync/core-stream.ts new file mode 100644 index 0000000000..e4f9c6bb89 --- /dev/null +++ b/src/packages/conat/sync/core-stream.ts @@ -0,0 +1,987 @@ +/* +core-stream.ts = the Core Stream data structure for conat. + +This is the core data structure that easy-to-use ephemeral and persistent +streams and kv stores are built on. It is NOT meant to be super easy and +simple to use, with save in the background. Instead, operations +are async, and the API is complicated. We build dkv, dstream, akv, etc. on +top of this with a much friendly API. + +NOTE: unlike in conat, in kv mode, the keys can be any utf-8 string. +We use the subject to track communication involving this stream, but +otherwise it has no relevant to the keys. Conat's core pub/sub/request/ +reply model is very similar to NATS, but the analogue of Jetstream is +different because I don't find Jetstream useful at all, and find this +much more useful. + +DEVELOPMENT: + +~/cocalc/src/packages/backend$ node + + require('@cocalc/backend/conat'); a = require('@cocalc/conat/sync/core-stream'); s = await a.cstream({name:'test'}) + +*/ + +import { EventEmitter } from "events"; +import { + Message, + type Headers, + messageData, + decode, +} from "@cocalc/conat/core/client"; +import { isNumericString } from "@cocalc/util/misc"; +import refCache from "@cocalc/util/refcache"; +import { conat } from "@cocalc/conat/client"; +import type { Client } from "@cocalc/conat/core/client"; +import jsonStableStringify from "json-stable-stringify"; +import type { + SetOperation, + DeleteOperation, + StoredMessage, + Configuration, +} from "@cocalc/conat/persist/storage"; +export type { Configuration }; +import { join } from "path"; +import { + type StorageOptions, + type PersistStreamClient, + stream as persist, + type SetOptions, +} from "@cocalc/conat/persist/client"; +import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; +import { until } from "@cocalc/util/async-utils"; +import { type PartialInventory } from "@cocalc/conat/persist/storage"; +import { getLogger } from "@cocalc/conat/client"; + +const logger = getLogger("sync:core-stream"); + +const PUBLISH_MANY_BATCH_SIZE = 500; + +const log = (..._args) => {}; +//const log = console.log; + +// when this many bytes of key:value have been changed (so need to be freed), +// we do a garbage collection pass. +export const KEY_GC_THRESH = 10 * 1e6; + +// NOTE: when you do delete this.deleteKv(key), we ensure the previous +// messages with the given key is completely deleted from sqlite, and +// also create a *new* lightweight tombstone. That tombstone has this +// ttl, which defaults to DEFAULT_TOMBSTONE_TTL (one week), so the tombstone +// itself will be removed after 1 week. The tombstone is only needed for +// clients that go offline during the delete, then come back, and reply the +// partial log of what was missed. Such clients should reset if the +// offline time is longer than DEFAULT_TOMBSTONE_TTL. +// This only happens if allow_msg_ttl is configured to true, which is +// done with dkv, but not on by default otherwise. +export const DEFAULT_TOMBSTONE_TTL = 7 * 24 * 60 * 60 * 1000; // 1 week + +export interface RawMsg extends Message { + timestamp: number; + seq: number; + key?: string; +} + +export interface ChangeEvent { + mesg?: T; + raw?: Partial; + key?: string; + prev?: T; + msgID?: string; +} + +const HEADER_PREFIX = "CN-"; + +export const COCALC_TOMBSTONE_HEADER = `${HEADER_PREFIX}Tombstone`; +export const COCALC_STREAM_HEADER = `${HEADER_PREFIX}Stream`; +export const COCALC_OPTIONS_HEADER = `${HEADER_PREFIX}Options`; + +export interface CoreStreamOptions { + // what it's called + name: string; + // where it is located -- this is who **owns the resource**, which + // may or may not being who is accessing it. + account_id?: string; + project_id?: string; + config?: Partial; + // only load historic messages starting at the given seq number. + start_seq?: number; + + ephemeral?: boolean; + + client?: Client; + + noCache?: boolean; +} + +export interface User { + account_id?: string; + project_id?: string; +} + +export function storagePath({ + account_id, + project_id, + name, +}: User & { name: string }) { + let userPath; + if (account_id) { + userPath = `accounts/${account_id}`; + } else if (project_id) { + userPath = `projects/${project_id}`; + } else { + userPath = "hub"; + } + return join(userPath, name); +} + +export class CoreStream extends EventEmitter { + public readonly name: string; + + private configOptions?: Partial; + private _start_seq?: number; + + // don't do "this.raw=" or "this.messages=" anywhere in this class + // because dstream directly references the public raw/messages. + public readonly raw: RawMsg[] = []; + public readonly messages: T[] = []; + public readonly kv: { [key: string]: { mesg: T; raw: RawMsg } } = {}; + private kvChangeBytes = 0; + + // this msgID's is ONLY used in ephemeral mode by the leader. + private readonly msgIDs = new Set(); + // lastSeq used by clients to keep track of what they have received; if one + // is skipped they reconnect starting with the last one they didn't miss. + private lastSeq: number = 0; + // IMPORTANT: user here means the *owner* of the resource, **NOT** the + // client who is accessing it! For example, a stream of edits of a file + // in a project has user {project_id} even if it is being accessed by + // an account. + private user: User; + private storage: StorageOptions; + private client?: Client; + private persistClient: PersistStreamClient; + + constructor({ + name, + project_id, + account_id, + config, + start_seq, + ephemeral = false, + client, + }: CoreStreamOptions) { + super(); + logger.debug("constructor", name) + if (client == null) { + throw Error("client must be specified"); + } + this.client = client; + this.user = { account_id, project_id }; + this.name = name; + this.storage = { + path: storagePath({ account_id, project_id, name }), + ephemeral, + }; + this._start_seq = start_seq; + this.configOptions = config; + return new Proxy(this, { + get(target, prop) { + return typeof prop == "string" && isNumericString(prop) + ? target.get(parseInt(prop)) + : target[String(prop)]; + }, + }); + } + + private initialized = false; + init = async () => { + if (this.initialized) { + throw Error("init can only be called once"); + } + this.initialized = true; + if (this.client == null) { + this.client = await conat(); + } + this.persistClient = persist({ + client: this.client, + user: this.user, + storage: this.storage, + }); + this.persistClient.on("error", (err) => { + if (!process.env.COCALC_TEST_MODE) { + console.log(`WARNING: persistent stream issue -- ${err}`); + } + }); + await this.getAllFromPersist({ + start_seq: this._start_seq, + noEmit: true, + }); + + await until( + async () => { + if (this.client == null) { + return true; + } + try { + this.configOptions = await this.config(this.configOptions); + return true; + } catch (err) { + if (err.code == 403) { + // fatal permission error + throw err; + } + } + return false; + }, + { start: 750 }, + ); + + // NOTE: if we miss a message between getAllFromLeader and when we start listening, + // it will get filled in, due to sequence number tracking. + this.listen(); + }; + + config = async ( + config: Partial = {}, + ): Promise => { + if (this.storage == null) { + throw Error("bug -- storage must be set"); + } + return await this.persistClient.config({ config }); + }; + + close = () => { + logger.debug("close", this.name) + delete this.client; + this.removeAllListeners(); + this.persistClient?.close(); + // @ts-ignore + delete this.persistClient; + // @ts-ignore + delete this.kv; + // @ts-ignore + delete this.messages; + // @ts-ignore + delete this.raw; + // @ts-ignore + delete this.msgIDs; + // @ts-ignore + delete this.storage; + }; + + inventory = async (): Promise => { + return await this.persistClient.inventory(); + }; + + // NOTE: It's assumed elsewhere that getAllFromPersist will not throw, + // and will keep retrying until (1) it works, or (2) self is closed, + // or (3) there is a fatal failure, e.g., lack of permissions. + private getAllFromPersist = async ({ + start_seq = 0, + noEmit, + }: { start_seq?: number; noEmit?: boolean } = {}) => { + if (this.storage == null) { + throw Error("bug -- storage must be set"); + } + await until( + async () => { + if (this.client == null) { + return true; + } + try { + // console.log("get persistent stream", { start_seq }, this.storage); + const sub = await this.persistClient.getAll({ + start_seq, + }); + // console.log("got sub"); + while (true) { + const { value, done } = await sub.next(); + if (done) { + return true; + } + const messages = value as StoredMessage[]; + const seq = this.processPersistentMessages(messages, { + noEmit, + noSeqCheck: true, + }); + if (seq != null) { + // we update start_seq in case we need to try again + start_seq = seq! + 1; + } + } + } catch (err) { + if (err.code == 403) { + // fatal permission error + throw err; + } + if (err.code == 429) { + // too many users + throw err; + } + if (!process.env.COCALC_TEST_MODE) { + console.log( + `WARNING: getAllFromPersist - failed -- ${err}, code=${err.code}`, + ); + } + } + return false; + }, + { start: 750 }, + ); + }; + + private processPersistentMessages = ( + messages: (SetOperation | DeleteOperation | StoredMessage)[], + opts: { noEmit?: boolean; noSeqCheck?: boolean }, + ) => { + // console.log("processPersistentMessages", messages.length, " messages"); + if (this.raw === undefined) { + // closed + return; + } + let seq = undefined; + for (const mesg of messages) { + try { + this.processPersistentMessage(mesg, opts); + if (mesg["seq"] != null) { + seq = mesg["seq"]; + } + } catch (err) { + console.log("WARNING: issue processing message", mesg, err); + } + } + return seq; + }; + + private processPersistentMessage = ( + mesg: SetOperation | DeleteOperation | StoredMessage, + opts: { noEmit?: boolean; noSeqCheck?: boolean }, + ) => { + if ((mesg as DeleteOperation).op == "delete") { + this.processPersistentDelete(mesg as DeleteOperation, opts); + } else { + // set is the default + this.processPersistentSet(mesg as SetOperation, opts); + } + }; + + private processPersistentDelete = ( + { seqs }: DeleteOperation, + { noEmit }: { noEmit?: boolean }, + ) => { + if (this.raw == null) return; + //console.log("processPersistentDelete", seqs); + const X = new Set(seqs); + // seqs is a list of integers. We remove + // every entry from this.raw, this.messages, and this.kv + // where this.raw.seq is in X by mutating raw/messages/kv, + // not by making new objects (since external references). + // This is a rare operation so we're not worried too much + // about performance. + const keys: { [seq: number]: string } = {}; + for (const key in this.kv) { + const seq = this.kv[key]?.raw?.seq; + if (X.has(seq)) { + delete this.kv[key]; + keys[key] = seq; + } + } + const indexes: number[] = []; + for (let i = 0; i < this.raw.length; i++) { + const seq = this.raw[i].seq; + if (X.has(seq)) { + indexes.push(i); + if (!noEmit) { + this.emitChange({ + mesg: undefined, + raw: { seq }, + key: keys[seq], + prev: this.messages[i], + }); + } + } + } + + //console.log({ indexes, seqs, noEmit }); + // remove this.raw[i] and this.messages[i] for all i in indexes, + // with special case to be fast in the very common case of contiguous indexes. + if (indexes.length > 1 && indexes.every((v, i) => v === indexes[0] + i)) { + // Contiguous: bulk remove for performance + const start = indexes[0]; + const deleteCount = indexes.length; + this.raw.splice(start, deleteCount); + this.messages.splice(start, deleteCount); + } else { + // Non-contiguous: fallback to individual reverse splices + for (let i = indexes.length - 1; i >= 0; i--) { + const idx = indexes[i]; + this.raw.splice(idx, 1); + this.messages.splice(idx, 1); + } + } + }; + + private processPersistentSetLargestSeq: number = 0; + private missingMessages = new Set(); + private processPersistentSet = ( + { seq, time, key, encoding, raw: data, headers, msgID }: SetOperation, + { + noEmit, + noSeqCheck, + }: { + noEmit?: boolean; + noSeqCheck?: boolean; + }, + ) => { + if (this.raw == null) return; + if (!noSeqCheck && this.processPersistentSetLargestSeq > 0) { + const expected = this.processPersistentSetLargestSeq + 1; + if (seq > expected) { + log( + "processPersistentSet -- detected missed seq number", + { seq, expected: this.processPersistentSetLargestSeq + 1 }, + this.storage, + ); + // We record that some are missing. + for (let s = expected; s <= seq - 1; s++) { + this.missingMessages.add(s); + this.getAllMissingMessages(); + } + } + } + + if (seq > this.processPersistentSetLargestSeq) { + this.processPersistentSetLargestSeq = seq; + } + + const mesg = decode({ encoding, data }); + const raw = { + timestamp: time, + headers, + seq, + raw: data, + key, + } as RawMsg; + if (seq > (this.raw.slice(-1)[0]?.seq ?? 0)) { + // easy fast initial load to the end of the list (common special case) + this.messages.push(mesg); + this.raw.push(raw); + } else { + // [ ] TODO: insert in the correct place. This should only + // happen when calling load of old ata. The algorithm below is + // dumb and could be replaced by a binary search. However, we'll + // change how we batch load so there's less point. + let i = 0; + while (i < this.raw.length && this.raw[i].seq < seq) { + i += 1; + } + this.raw.splice(i, 0, raw); + this.messages.splice(i, 0, mesg); + } + let prev: T | undefined = undefined; + if (typeof key == "string") { + prev = this.kv[key]?.mesg; + if (raw.headers?.[COCALC_TOMBSTONE_HEADER]) { + delete this.kv[key]; + } else { + if (this.kv[key] !== undefined) { + const { raw } = this.kv[key]; + this.kvChangeBytes += raw.raw.length; + } + + this.kv[key] = { raw, mesg }; + + if (this.kvChangeBytes >= KEY_GC_THRESH) { + this.gcKv(); + } + } + } + this.lastSeq = Math.max(this.lastSeq, seq); + if (!noEmit) { + this.emitChange({ mesg, raw, key, prev, msgID }); + } + }; + + private emitChange = (e: ChangeEvent) => { + if (this.raw == null) return; + this.emit("change", e); + }; + + private listen = async () => { + log("core-stream: listen", this.storage); + await until( + async () => { + if (this.client == null) { + return true; + } + try { + log("core-stream: START listening on changefeed", this.storage); + const changefeed = await this.persistClient.changefeed(); + for await (const { updates } of changefeed) { + log("core-stream: process updates", updates, this.storage); + if (this.client == null) return true; + this.processPersistentMessages(updates, { + noEmit: false, + noSeqCheck: false, + }); + } + } catch (err) { + // This normally doesn't happen but could if a persist server is being restarted + // frequently or things are seriously broken. We cause this in + // backend/conat/test/core/core-stream-break.test.ts + if (!process.env.COCALC_TEST_MODE) { + log( + `WARNING: core-stream changefeed error -- ${err}`, + this.storage, + ); + } + } + log("core-stream: STOP listening on changefeed", this.storage); + // above loop exits when the persistent server + // stops sending messages for some reason. In that + // case we reconnect, picking up where we left off: + if (this.client == null) return true; + log( + "core-stream: get missing from when changefeed ended", + this.storage, + ); + await this.getAllFromPersist({ + start_seq: this.lastSeq + 1, + noEmit: false, + }); + return false; + }, + { start: 500, max: 7500, decay: 1.2 }, + ); + }; + + publish = async ( + mesg: T, + options?: PublishOptions, + ): Promise<{ seq: number; time: number } | undefined> => { + if (mesg === undefined) { + if (options?.key !== undefined) { + // undefined can't be JSON encoded, so we can't possibly represent it, and this + // *must* be treated as a delete. + this.deleteKv(options?.key, { previousSeq: options?.previousSeq }); + return; + } else { + throw Error("stream non-kv publish - mesg must not be 'undefined'"); + } + } + + if (options?.msgID && this.msgIDs.has(options.msgID)) { + // it's a dup + return; + } + const md = messageData(mesg, { headers: options?.headers }); + const x = await this.persistClient.set({ + key: options?.key, + messageData: md, + previousSeq: options?.previousSeq, + msgID: options?.msgID, + ttl: options?.ttl, + timeout: options?.timeout, + }); + if (options?.msgID) { + this.msgIDs?.add(options.msgID); + } + return x; + }; + + publishMany = async ( + messages: { mesg: T; options?: PublishOptions }[], + ): Promise< + ({ seq: number; time: number } | { error: string; code?: any })[] + > => { + let result: ( + | { seq: number; time: number } + | { error: string; code?: any } + )[] = []; + + for (let i = 0; i < messages.length; i += PUBLISH_MANY_BATCH_SIZE) { + const batch = messages.slice(i, i + PUBLISH_MANY_BATCH_SIZE); + result = result.concat(await this.publishMany0(batch)); + } + + return result; + }; + + private publishMany0 = async ( + messages: { mesg: T; options?: PublishOptions }[], + ): Promise< + ({ seq: number; time: number } | { error: string; code?: any })[] + > => { + const v: SetOptions[] = []; + let timeout: number | undefined = undefined; + for (const { mesg, options } of messages) { + if (options?.timeout) { + if (timeout === undefined) { + timeout = options.timeout; + } else { + timeout = Math.min(timeout, options.timeout); + } + } + const md = messageData(mesg, { headers: options?.headers }); + v.push({ + key: options?.key, + messageData: md, + previousSeq: options?.previousSeq, + msgID: options?.msgID, + ttl: options?.ttl, + }); + } + return await this.persistClient.setMany(v, { timeout }); + }; + + get = (n?): T | T[] => { + if (n == null) { + return this.getAll(); + } else { + return this.messages[n]; + } + }; + + seq = (n: number): number | undefined => { + return this.raw[n]?.seq; + }; + + getAll = (): T[] => { + return [...this.messages]; + }; + + get length(): number { + return this.messages.length; + } + + get start_seq(): number | undefined { + return this._start_seq; + } + + headers = (n: number): { [key: string]: any } | undefined => { + return this.raw[n]?.headers; + }; + + // key:value interface for subset of messages pushed with key option set. + // NOTE: This does NOT throw an error if our local seq is out of date (leave that + // to dkv built on this). + setKv = async ( + key: string, + mesg: T, + options?: { + headers?: Headers; + previousSeq?: number; + }, + ): Promise<{ seq: number; time: number } | undefined> => { + return await this.publish(mesg, { ...options, key }); + }; + + setKvMany = async ( + x: { + key: string; + mesg: T; + options?: { + headers?: Headers; + previousSeq?: number; + }; + }[], + ): Promise< + ({ seq: number; time: number } | { error: string; code?: any })[] + > => { + const messages: { mesg: T; options?: PublishOptions }[] = []; + for (const { key, mesg, options } of x) { + messages.push({ mesg, options: { ...options, key } }); + } + return await this.publishMany(messages); + }; + + deleteKv = async ( + key: string, + options?: { + msgID?: string; + previousSeq?: number; + }, + ) => { + if (this.kv[key] === undefined) { + // nothing to do + return; + } + return await this.publish(null as any, { + ...options, + headers: { [COCALC_TOMBSTONE_HEADER]: true }, + key, + ttl: DEFAULT_TOMBSTONE_TTL, + }); + }; + + getKv = (key: string): T | undefined => { + return this.kv[key]?.mesg; + }; + + hasKv = (key: string): boolean => { + return this.kv?.[key] !== undefined; + }; + + getAllKv = (): { [key: string]: T } => { + const all: { [key: string]: T } = {}; + for (const key in this.kv) { + all[key] = this.kv[key].mesg; + } + return all; + }; + + seqKv = (key: string): number | undefined => { + return this.kv[key]?.raw.seq; + }; + + timeKv = (key?: string): Date | { [key: string]: Date } | undefined => { + if (key === undefined) { + const all: { [key: string]: Date } = {}; + for (const key in this.kv) { + all[key] = new Date(this.kv[key].raw.timestamp); + } + return all; + } + const r = this.kv[key]?.raw; + if (r == null) { + return; + } + return new Date(r.timestamp); + }; + + headersKv = (key: string): { [key: string]: any } | undefined => { + return this.kv[key]?.raw?.headers; + }; + + get lengthKv(): number { + return Object.keys(this.kv).length; + } + + // load older messages starting at start_seq up to the oldest message + // we currently have. + load = async ({ + start_seq, + noEmit, + }: { + start_seq: number; + noEmit?: boolean; + }) => { + // This is used for loading more TimeTravel history + if (this.storage == null) { + throw Error("bug"); + } + // this is one before the oldest we have + const end_seq = (this.raw[0]?.seq ?? this._start_seq ?? 1) - 1; + if (start_seq > end_seq) { + // nothing to load + return; + } + // we're moving start_seq back to this point + this._start_seq = start_seq; + const sub = await this.persistClient.getAll({ + start_seq, + end_seq, + }); + for await (const updates of sub) { + this.processPersistentMessages(updates, { noEmit, noSeqCheck: true }); + } + }; + + private getAllMissingMessages = reuseInFlight(async () => { + await until( + async () => { + if (this.client == null || this.missingMessages.size == 0) { + return true; + } + try { + const missing = Array.from(this.missingMessages); + missing.sort(); + log("core-stream: getMissingSeq", missing, this.storage); + const sub = await this.persistClient.getAll({ + start_seq: missing[0], + end_seq: missing[missing.length - 1], + }); + for await (const updates of sub) { + this.processPersistentMessages(updates, { + noEmit: false, + noSeqCheck: true, + }); + } + for (const seq of missing) { + this.missingMessages.delete(seq); + } + } catch (err) { + log( + "core-stream: WARNING -- issue getting missing updates", + err, + this.storage, + ); + } + return false; + }, + { start: 1000, max: 15000, decay: 1.3 }, + ); + }); + + // get server assigned time of n-th message in stream + time = (n: number): Date | undefined => { + const r = this.raw[n]; + if (r == null) { + return; + } + return new Date(r.timestamp); + }; + + times = () => { + const v: (Date | undefined)[] = []; + for (let i = 0; i < this.length; i++) { + v.push(this.time(i)); + } + return v; + }; + + stats = ({ + start_seq = 1, + }: { + start_seq?: number; + } = {}): { count: number; bytes: number } | undefined => { + if (this.raw == null) { + return; + } + let count = 0; + let bytes = 0; + for (const { raw, seq } of this.raw) { + if (seq == null) { + continue; + } + if (seq < start_seq) { + continue; + } + count += 1; + bytes += raw.length; + } + return { count, bytes }; + }; + + // delete all messages up to and including the + // one at position index, i.e., this.messages[index] + // is deleted. + // NOTE: For ephemeral streams, clients will NOT see the result of a delete, + // except when they load the stream later. For persistent streams all + // **connected** clients will see the delete. THAT said, this is not a "proper" + // distributed computing primitive with tombstones, etc. This is primarily + // meant for reducing space usage, and shouldn't be relied on for + // any other purpose. + delete = async ({ + all, + last_index, + seq, + last_seq, + key, + }: { + // give exactly ONE parameter -- by default nothing happens with no params + // all: delete everything + all?: boolean; + // last_index: everything up to and including index'd message + last_index?: number; + // seq: delete message with this sequence number + seq?: number; + // last_seq: delete everything up to and including this sequence number + last_seq?: number; + // key: delete the message with this key + key?: string; + } = {}): Promise<{ seqs: number[] }> => { + let opts; + if (all) { + opts = { all: true }; + } else if (last_index != null) { + if (last_index >= this.raw.length) { + opts = { all: true }; + } else if (last_index < 0) { + return { seqs: [] }; + } else { + const last_seq = this.raw[last_index].seq; + if (last_seq === undefined) { + throw Error(`BUG: invalid index ${last_index}`); + } + opts = { last_seq }; + } + } else if (seq != null) { + opts = { seq }; + } else if (last_seq != null) { + opts = { last_seq }; + } else if (key != null) { + const seq = this.kv[key]?.raw?.seq; + if (seq === undefined) { + return { seqs: [] }; + } + opts = { seq }; + } + return await this.persistClient.delete(opts); + }; + + // delete messages that are no longer needed since newer values have been written + gcKv = () => { + this.kvChangeBytes = 0; + for (let i = 0; i < this.raw.length; i++) { + const key = this.raw[i].key; + if (key !== undefined) { + if (this.raw[i].raw.length > 0 && this.raw[i] !== this.kv[key].raw) { + this.raw[i] = { + ...this.raw[i], + headers: undefined, + raw: Buffer.from(""), + } as RawMsg; + this.messages[i] = undefined as T; + } + } + } + }; +} + +export interface PublishOptions { + // headers for this message + headers?: Headers; + // unique id for this message to dedup so if you send the same + // message more than once with the same id it doesn't get published + // multiple times. + msgID?: string; + // key -- if specified a key field is also stored on the server, + // and any previous messages with the same key are deleted. Also, + // an entry is set in this.kv[key] so that this.getKv(key), etc. work. + key?: string; + // if key is specified and previousSeq is set, the server throws + // an error if the sequence number of the current key is + // not previousSeq. We use this with this.seqKv(key) to + // provide read/change/write semantics and to know when we + // should resovle a merge conflict. This is ignored if + // key is not specified. + previousSeq?: number; + // if set to a number of ms AND the config option allow_msg_ttl + // is set on this persistent stream, then + // this message will be deleted after the given amount of time (in ms). + ttl?: number; + timeout?: number; +} + +export const cache = refCache({ + name: "core-stream", + createObject: async (options: CoreStreamOptions) => { + if (options.client == null) { + options = { ...options, client: await conat() }; + } + const cstream = new CoreStream(options); + await cstream.init(); + return cstream; + }, + createKey: ({ client, ...options }) => { + return jsonStableStringify(options)!; + }, +}); + +export async function cstream( + options: CoreStreamOptions, +): Promise> { + return await cache(options); +} diff --git a/src/packages/nats/sync/dko.ts b/src/packages/conat/sync/dko.ts similarity index 64% rename from src/packages/nats/sync/dko.ts rename to src/packages/conat/sync/dko.ts index 3247bd6a62..70504a0f99 100644 --- a/src/packages/nats/sync/dko.ts +++ b/src/packages/conat/sync/dko.ts @@ -1,34 +1,43 @@ /* Distributed eventually consistent key:object store, where changes propogate sparsely. -The "values" MUST be objects and no keys or fields of objects can container the sep character, -which is '|' by default. +The "values" MUST be objects and no keys or fields of objects can container the +sep character, which is '|' by default. + +NOTE: Whenever you do a set, the lodash isEqual function is used to see which fields +you are setting are actually different, and only those get sync'd out. +This takes more resources on each client, but less on the network and servers. +It also means that if two clients write to an object at the same time but to +different field (a merge conflict), then the result gets merged together properly +with last write wins per field. DEVELOPMENT: ~/cocalc/src/packages/backend n -Welcome to Node.js v18.17.1. -Type ".help" for more information. -> t = await require("@cocalc/backend/nats/sync").dko({name:'test'}) + > t = await require("@cocalc/backend/conat/sync").dko({name:'test'}) */ import { EventEmitter } from "events"; -import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { dkv as createDKV, DKV, DKVOptions } from "./dkv"; -import { userKvKey } from "./kv"; import { is_object } from "@cocalc/util/misc"; import refCache from "@cocalc/util/refcache"; -import { getEnv } from "@cocalc/nats/client"; +import jsonStableStringify from "json-stable-stringify"; +import { isEqual } from "lodash"; + +export function userKvKey(options: DKVOptions) { + if (!options.name) { + throw Error("name must be specified"); + } + const { client, ...x } = options; + return jsonStableStringify(x)!; +} export class DKO extends EventEmitter { - opts: DKVOptions; dkv?: DKV; // can't type this - constructor(opts: DKVOptions) { + constructor(private opts: DKVOptions) { super(); - this.opts = opts; - this.init(); return new Proxy(this, { deleteProperty(target, prop) { if (typeof prop == "string") { @@ -55,56 +64,64 @@ export class DKO extends EventEmitter { }); } - init = reuseInFlight(async () => { - if (this.dkv != null) { - throw Error("already initialized"); + private dkvOnChange = ({ key: path, value }) => { + if (path == null) { + // TODO: could this happen? + return; } - this.dkv = await createDKV<{ [key: string]: any }>({ - ...this.opts, - name: dkoPrefix(this.opts.name), - }); - this.dkv.on("change", ({ key: path, value }) => { - if (path == null) { - // TODO: could this happen? + const { key, field } = this.fromPath(path); + if (!field) { + // there is no field part of the path, which happens + // only for delete of entire object, after setting all + // the fields to null. + this.emit("change", { key }); + } else { + if (value === undefined && this.dkv?.get(key) == null) { + // don't emit change setting fields to undefined if the + // object was already deleted. return; } - const { key, field } = this.fromPath(path); - if (!field) { - // there is no field part of the path, which happens - // only for delete of entire object, after setting all - // the fields to null. - this.emit("change", { key }); - } else { - if (value === undefined && this.dkv?.get(key) == null) { - // don't emit change setting fields to undefined if the - // object was already deleted. - return; - } - this.emit("change", { key, field, value }); - } - }); + this.emit("change", { key, field, value }); + } + }; - this.dkv.on("reject", ({ key: path, value }) => { - if (path == null) { - // TODO: would this happen? - return; - } - const { key, field } = this.fromPath(path); - if (!field) { - this.emit("reject", { key }); - } else { - this.emit("reject", { key, field, value }); - } + private dkvOnReject = ({ key: path, value }) => { + if (path == null) { + // TODO: would this happen? + return; + } + const { key, field } = this.fromPath(path); + if (!field) { + this.emit("reject", { key }); + } else { + this.emit("reject", { key, field, value }); + } + }; + + private initialized = false; + init = async () => { + if (this.initialized) { + throw Error("init can only be called once"); + } + this.initialized = true; + this.dkv = await createDKV<{ [key: string]: any }>({ + ...this.opts, + name: dkoPrefix(this.opts.name), }); - await this.dkv.init(); - }); + this.dkv.on("change", this.dkvOnChange); + this.dkv.on("reject", this.dkvOnReject); + }; close = async () => { if (this.dkv == null) { return; } + this.dkv.removeListener("change", this.dkvOnChange); + this.dkv.removeListener("reject", this.dkvOnReject); await this.dkv.close(); delete this.dkv; + // @ts-ignore + delete this.opts; this.emit("closed"); this.removeAllListeners(); }; @@ -192,9 +209,17 @@ export class DKO extends EventEmitter { throw Error("values must be objects"); } const fields = Object.keys(obj); - this.dkv.set(key, fields); + const cur = this.dkv.get(key); + if (!isEqual(cur, fields)) { + this.dkv.set(key, fields); + } for (const field of fields) { - this.dkv.set(this.toPath(key, field), obj[field]); + const path = this.toPath(key, field); + const value = obj[field]; + const cur = this.dkv.get(path); + if (!isEqual(cur, value)) { + this.dkv.set(path, value); + } } }; @@ -227,9 +252,6 @@ export const cache = refCache({ name: "dko", createKey: userKvKey, createObject: async (opts) => { - if (opts.env == null) { - opts.env = await getEnv(); - } const k = new DKO(opts); await k.init(); return k; diff --git a/src/packages/conat/sync/dkv.ts b/src/packages/conat/sync/dkv.ts new file mode 100644 index 0000000000..c3f6abc761 --- /dev/null +++ b/src/packages/conat/sync/dkv.ts @@ -0,0 +1,816 @@ +/* +Eventually Consistent Distributed Key:Value Store + +- You give one subject and general-dkv provides a synchronous eventually consistent + "multimaster" distributed way to work with the KV store of keys matching that subject, + inside of the named KV store. + +- You may define a 3-way merge function, which is used to automatically resolve all + conflicting writes. The default is to use our local version, i.e., "last write + to remote wins". The function is run locally so can have access to any state. + +- All set/get/delete operations are synchronous. + +- The state gets sync'd in the backend to persistent storage on Conat as soon as possible, + and there is an async save function. + +This class is based on top of the Consistent Centralized Key:Value Store defined in kv.ts. +You can use the same key:value store at the same time via both interfaces, and if the store +is a DKV, you can also access the underlying KV via "store.kv". + +- You must explicitly call "await store.init()" to initialize this before using it. + +- The store emits an event ('change', key) whenever anything changes. + +- Calling "store.getAll()" provides ALL the data, and "store.get(key)" gets one value. + +- Use "store.set(key,value)" or "store.set({key:value, key2:value2, ...})" to set data, + with the following semantics: + + - in the background, changes propagate to Conat. You do not do anything explicitly and + this should never raise an exception. + + - you can call "store.hasUnsavedChanges()" to see if there are any unsaved changes. + + - call "store.unsavedChanges()" to see the unsaved keys. + +- The 3-way merge function takes as input {local,remote,prev,key}, where + - key = the key where there's a conflict + - local = your version of the value + - remote = the remote value, which conflicts in that isEqual(local,remote) is false. + - prev = a known common prev of local and remote. + + (any of local, remote or prev can be undefined, e.g., no previous value or a key was deleted) + + You can do anything synchronously you want to resolve such conflicts, i.e., there are no + axioms that have to be satisifed. If the 3-way merge function throws an exception (or is + not specified) we silently fall back to "last write wins". + + +DEVELOPMENT: + +~/cocalc/src/packages/backend$ node + +s = await require("@cocalc/backend/conat/sync").dkv({name:'test', merge:({local,remote})=>{return {...remote,...local}}}); + + +In the browser console: + +> s = await cc.client.conat_client.dkv({filter:['foo.>'],merge:({local,remote})=>{return {...remote,...local}}}) + +# NOTE that the name is account-{account_id} or project-{project_id}, +# and if not given defaults to the account-{user's account id} +> s.kv.name +'account-6aae57c6-08f1-4bb5-848b-3ceb53e61ede' + +> s.on('change',(key)=>console.log(key));0; + +*/ + +import { EventEmitter } from "events"; +import { + CoreStream, + type Configuration, + type ChangeEvent, +} from "./core-stream"; +import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; +import { isEqual } from "lodash"; +import { delay, map as awaitMap } from "awaiting"; +import { + type Client, + ConatError, + type Headers, +} from "@cocalc/conat/core/client"; +import refCache from "@cocalc/util/refcache"; +import { type JSONValue } from "@cocalc/util/types"; +import { conat } from "@cocalc/conat/client"; +import { asyncThrottle, until } from "@cocalc/util/async-utils"; +import { + inventory, + type Inventory, + INVENTORY_UPDATE_INTERVAL, +} from "./inventory"; + +export const TOMBSTONE = Symbol("tombstone"); +const MAX_PARALLEL = 250; + +const DEBUG = false; + +export type MergeFunction = (opts: { + key: string; + prev: any; + local: any; + remote: any; +}) => any; + +interface SetOptions { + headers?: Headers; +} + +export interface DKVOptions { + name: string; + account_id?: string; + project_id?: string; + desc?: JSONValue; + client?: Client; + // 3-way merge conflict resolution + merge?: (opts: { key: string; prev?: any; local?: any; remote?: any }) => any; + config?: Partial; + + // if noAutosave is set, local changes are never saved until you explicitly + // call "await this.save()", which will try once to save. Changes made during + // the save may not be saved though. + // CAUTION: noAutosave is really only meant for unit testing! The save is + // reuseInFlighted so a safe somewhere far away could be in progress starting + // before your call to save, and when it finishes that's it, so what you just + // did is not saved. Take care. + noAutosave?: boolean; + + ephemeral?: boolean; + + noCache?: boolean; + noInventory?: boolean; +} + +export class DKV extends EventEmitter { + private kv?: CoreStream; + private merge?: MergeFunction; + private local: { [key: string]: T | typeof TOMBSTONE } = {}; + private options: { [key: string]: SetOptions } = {}; + private saved: { [key: string]: T | typeof TOMBSTONE } = {}; + private changed: Set = new Set(); + private noAutosave: boolean; + public readonly name: string; + public readonly desc?: JSONValue; + private saveErrors: boolean = false; + private invalidSeq = new Set(); + private opts: DKVOptions; + + constructor(opts: DKVOptions) { + super(); + if (opts.client == null) { + throw Error("client must be specified"); + } + this.opts = opts; + const { + name, + project_id, + account_id, + desc, + client, + merge, + config, + noAutosave, + ephemeral = false, + } = opts; + this.name = name; + this.desc = desc; + this.merge = merge; + this.noAutosave = !!noAutosave; + this.kv = new CoreStream({ + name, + project_id, + account_id, + client, + config, + ephemeral, + }); + + return new Proxy(this, { + deleteProperty(target, prop) { + if (typeof prop == "string") { + target.delete(prop); + } + return true; + }, + set(target, prop, value) { + prop = String(prop); + if (prop == "_eventsCount" || prop == "_events" || prop == "close") { + target[prop] = value; + return true; + } + if (target[prop] != null) { + throw Error(`method name '${prop}' is read only`); + } + target.set(prop, value); + return true; + }, + get(target, prop) { + return target[String(prop)] ?? target.get(String(prop)); + }, + }); + } + + private initialized = false; + init = async () => { + if (this.initialized) { + throw Error("init can only be called once"); + } + this.initialized = true; + if (this.kv == null) { + throw Error("closed"); + } + this.kv.on("change", this.handleRemoteChange); + await this.kv.init(); + // allow_msg_ttl is used for deleting tombstones. + await this.kv.config({ allow_msg_ttl: true }); + this.emit("connected"); + }; + + isClosed = () => { + return this.kv == null; + }; + + close = () => { + if (this.isClosed()) { + return; + } + const kv = this.kv; + delete this.kv; + if (kv != null) { + kv.removeListener("change", this.handleRemoteChange); + kv.close(); + } + this.emit("closed"); + this.removeAllListeners(); + // @ts-ignore + delete this.local; + // @ts-ignore + delete this.options; + // @ts-ignore + delete this.changed; + delete this.merge; + // @ts-ignore + delete this.opts; + }; + + private discardLocalState = (key: string) => { + delete this.local[key]; + delete this.options[key]; + delete this.saved[key]; + if (this.isStable()) { + this.emit("stable"); + } + }; + + // stable = everything is saved *and* also echoed back from the server as confirmation. + isStable = () => { + for (const _ in this.local) { + return false; + } + return true; + }; + + private handleRemoteChange = ({ + mesg: remote, + key, + prev, + }: ChangeEvent) => { + if (key === undefined) { + // not part of kv store data + return; + } + const local = this.local[key] === TOMBSTONE ? undefined : this.local[key]; + let value: any = remote; + if (local !== undefined) { + // we have an unsaved local value, so let's check to see if there is a + // conflict or not. + if (isEqual(local, remote)) { + // incoming remote value is equal to unsaved local value, so we can + // just discard our local value (no need to save it). + this.discardLocalState(key); + } else { + // There is a conflict. Let's resolve the conflict: + // console.log("merge conflict", { key, remote, local, prev }); + try { + value = this.merge?.({ key, local, remote, prev }) ?? local; + // console.log("merge conflict --> ", value); + // console.log("handle merge conflict", { + // key, + // local, + // remote, + // prev, + // value, + // }); + } catch (err) { + console.warn("exception in merge conflict resolution", err); + // user provided a merge function that throws an exception. We select local, since + // it is the newest, i.e., "last write wins" + value = local; + // console.log("merge conflict ERROR --> ", err, value); + } + if (isEqual(value, remote)) { + // no change, so forget our local value + this.discardLocalState(key); + } else { + // resolve with the new value, or if it is undefined, a TOMBSTONE, + // meaning choice is to delete. + // console.log("conflict resolution: ", { key, value }); + if (value === TOMBSTONE) { + this.delete(key); + } else { + this.set(key, value); + } + } + } + } + this.emit("change", { key, value, prev }); + }; + + get(key: string): T | undefined; + get(): { [key: string]: T }; + get(key?: string): T | { [key: string]: T } | undefined { + if (this.kv == null) { + throw Error("closed"); + } + if (key === undefined) { + return this.getAll(); + } + const local = this.local[key]; + if (local === TOMBSTONE) { + return undefined; + } + if (local !== undefined) { + return local; + } + return this.kv.getKv(key); + } + + get length(): number { + // not efficient + return Object.keys(this.getAll()).length; + } + + getAll = (): { [key: string]: T } => { + if (this.kv == null) { + throw Error("closed"); + } + const x = { ...this.kv.getAllKv(), ...this.local }; + for (const key in this.local) { + if (this.local[key] === TOMBSTONE) { + delete x[key]; + } + } + return x as { [key: string]: T }; + }; + + keys = (): string[] => { + return Object.keys(this.getAll()); + }; + + has = (key: string): boolean => { + if (this.kv == null) { + throw Error("closed"); + } + const a = this.local[key]; + if (a === TOMBSTONE) { + return false; + } + if (a !== undefined) { + return true; + } + return this.kv.hasKv(key); + }; + + time = (key?: string): { [key: string]: Date } | Date | undefined => { + if (this.kv == null) { + throw Error("closed"); + } + return this.kv.timeKv(key); + }; + + seq = (key: string): number | undefined => { + if (this.kv == null) { + throw Error("closed"); + } + return this.kv.seqKv(key); + }; + + private _delete = (key) => { + this.local[key] = TOMBSTONE; + this.changed.add(key); + }; + + delete = (key) => { + this._delete(key); + if (!this.noAutosave) { + this.save(); + } + }; + + clear = () => { + if (this.kv == null) { + throw Error("closed"); + } + for (const key in this.kv.getAllKv()) { + this._delete(key); + } + for (const key in this.local) { + this._delete(key); + } + if (!this.noAutosave) { + this.save(); + } + }; + + private toValue = (obj) => { + if (obj === undefined) { + return TOMBSTONE; + } + return obj; + }; + + headers = (key: string): Headers | undefined => { + if (this.options[key] != null) { + return this.options[key]?.headers; + } else { + return this.kv?.headersKv(key); + } + }; + + set = (key: string, value: T, options?: SetOptions) => { + const obj = this.toValue(value); + this.local[key] = obj; + if (options != null) { + this.options[key] = options; + } + this.changed.add(key); + if (!this.noAutosave) { + this.save(); + } + this.updateInventory(); + }; + + setMany = (obj) => { + for (const key in obj) { + this.local[key] = this.toValue(obj[key]); + this.changed.add(key); + } + if (!this.noAutosave) { + this.save(); + } + this.updateInventory(); + }; + + hasUnsavedChanges = () => { + if (this.kv == null) { + return false; + } + return this.unsavedChanges().length > 0; + }; + + unsavedChanges = (): string[] => { + return Object.keys(this.local).filter( + (key) => this.local[key] !== this.saved[key], + ); + }; + + save = reuseInFlight(async () => { + if (this.noAutosave) { + return await this.attemptToSave(); + } + let status; + + await until( + async () => { + if (this.kv == null) { + return true; + } + try { + status = await this.attemptToSave(); + //console.log("successfully saved"); + } catch (err) { + if (!process.env.COCALC_TEST_MODE) { + console.log( + "WARNING: dkv attemptToSave failed -- ", + this.name, + this.kv?.name, + err, + ); + } + } + return !this.hasUnsavedChanges(); + }, + { start: 150, decay: 1.3, max: 10000 }, + ); + return status; + }); + + private attemptToSave = async () => { + if (true) { + await this.attemptToSaveMany(); + } else { + await this.attemptToSaveParallel(); + } + }; + + private attemptToSaveMany = reuseInFlight(async () => { + let start = Date.now(); + if (DEBUG) { + console.log("attemptToSaveMany: start"); + } + if (this.kv == null) { + throw Error("closed"); + } + this.changed.clear(); + const status = { unsaved: 0, set: 0, delete: 0 }; + const obj = { ...this.local }; + for (const key in obj) { + if (obj[key] === TOMBSTONE) { + status.unsaved += 1; + await this.kv.deleteKv(key); + if (this.kv == null) return; + status.delete += 1; + status.unsaved -= 1; + delete obj[key]; + if (!this.changed.has(key)) { + // successfully saved this and user didn't make a change *during* the set + this.discardLocalState(key); + } + } + } + let errors = false; + const x: { + key: string; + mesg: T; + options?: { + headers?: Headers; + previousSeq?: number; + }; + }[] = []; + for (const key in obj) { + const previousSeq = this.merge != null ? this.seq(key) : undefined; + if (previousSeq && this.invalidSeq.has(previousSeq)) { + continue; + } + status.unsaved += 1; + x.push({ + key, + mesg: obj[key] as T, + options: { + ...this.options[key], + previousSeq, + }, + }); + } + const results = await this.kv.setKvMany(x); + + let i = 0; + for (const resp of results) { + const { key } = x[i]; + i++; + if (this.kv == null) return; + if (!(resp as any).error) { + status.unsaved -= 1; + status.set += 1; + } else { + const { code, error } = resp as any; + if (DEBUG) { + console.log("kv store -- attemptToSave failed", this.desc, error, { + key, + value: obj[key], + code: code, + }); + } + errors = true; + if (code == "reject") { + const value = this.local[key]; + // can never save this. + this.discardLocalState(key); + status.unsaved -= 1; + this.emit("reject", { key, value }); + } + if (code == "wrong-last-sequence") { + // This happens when another client has published a NEWER version of this key, + // so the right thing is to just ignore this. In a moment there will be no + // need to save anything, since we'll receive a message that overwrites this key. + // It's very important that the changefeed actually be working, of course, which + // is why the this.invalidSeq, so we never retry in this case, since it can't work. + if (x[i]?.options?.previousSeq) { + this.invalidSeq.add(x[i].options!.previousSeq!); + } + return; + } + if (code == 408) { + // timeout -- expected to happen periodically, of course + if (!process.env.COCALC_TEST_MODE) { + console.log("WARNING: timeout saving (will try again soon)"); + } + return; + } + if (!process.env.COCALC_TEST_MODE) { + console.warn( + `WARNING: unexpected error saving dkv '${this.name}' -- ${error}`, + ); + } + } + } + if (errors) { + this.saveErrors = true; + throw Error(`there were errors saving dkv '${this.name}'`); + // so it retries + } else { + if ( + !process.env.COCALC_TEST_MODE && + this.saveErrors && + status.unsaved == 0 + ) { + this.saveErrors = false; + console.log(`SUCCESS: dkv ${this.name} fully saved`); + } + } + if (DEBUG) { + console.log("attemptToSaveMany: done", Date.now() - start); + } + + return status; + }); + + attemptToSaveParallel = reuseInFlight(async () => { + let start = Date.now(); + if (DEBUG) { + console.log("attemptToSaveParallel: start"); + } + if (this.kv == null) { + throw Error("closed"); + } + this.changed.clear(); + const status = { unsaved: 0, set: 0, delete: 0 }; + const obj = { ...this.local }; + for (const key in obj) { + if (obj[key] === TOMBSTONE) { + status.unsaved += 1; + await this.kv.deleteKv(key); + if (this.kv == null) return; + status.delete += 1; + status.unsaved -= 1; + delete obj[key]; + if (!this.changed.has(key)) { + // successfully saved this and user didn't make a change *during* the set + this.discardLocalState(key); + } + } + } + let errors = false; + const f = async (key: string) => { + if (this.kv == null) { + // closed + return; + } + const previousSeq = this.merge != null ? this.seq(key) : undefined; + try { + if (previousSeq && this.invalidSeq.has(previousSeq)) { + throw new ConatError("waiting on new sequence via changefeed", { + code: "wrong-last-sequence", + }); + } + status.unsaved += 1; + await this.kv.setKv(key, obj[key] as T, { + ...this.options[key], + previousSeq, + }); + if (this.kv == null) return; + if (DEBUG) { + console.log("kv store -- attemptToSave succeed", this.desc, { + key, + value: obj[key], + }); + } + status.unsaved -= 1; + status.set += 1; + // note that we CANNOT call this.discardLocalState(key) here, because + // this.get(key) needs to work immediately after save, but if this.local[key] + // is deleted, then this.get(key) would be undefined, because + // this.kv.getKv(key) only has value in it once the value is + // echoed back from the server. + } catch (err) { + if (DEBUG) { + console.log("kv store -- attemptToSave failed", this.desc, err, { + key, + value: obj[key], + code: err.code, + }); + } + errors = true; + if (err.code == "reject") { + const value = this.local[key]; + // can never save this. + this.discardLocalState(key); + status.unsaved -= 1; + this.emit("reject", { key, value }); + } + if (err.code == "wrong-last-sequence") { + // This happens when another client has published a NEWER version of this key, + // so the right thing is to just ignore this. In a moment there will be no + // need to save anything, since we'll receive a message that overwrites this key. + // It's very important that the changefeed actually be working, of course, which + // is why the this.invalidSeq, so we never retry in this case, since it can't work. + if (previousSeq) { + this.invalidSeq.add(previousSeq); + } + return; + } + if (err.code == 408) { + // timeout -- expected to happen periodically, of course + if (!process.env.COCALC_TEST_MODE) { + console.log("WARNING: timeout saving (will try again soon)"); + } + return; + } + if (!process.env.COCALC_TEST_MODE) { + console.warn( + `WARNING: unexpected error saving dkv '${this.name}' -- ${err}`, + ); + } + } + }; + await awaitMap(Object.keys(obj), MAX_PARALLEL, f); + if (errors) { + this.saveErrors = true; + throw Error(`there were errors saving dkv '${this.name}'`); + // so it retries + } else { + if ( + !process.env.COCALC_TEST_MODE && + this.saveErrors && + status.unsaved == 0 + ) { + this.saveErrors = false; + console.log(`SUCCESS: dkv ${this.name} fully saved`); + } + } + if (DEBUG) { + console.log("attemptToSaveParallel: done", Date.now() - start); + } + + return status; + }); + + stats = () => this.kv?.stats(); + + // get or set config + config = async ( + config: Partial = {}, + ): Promise => { + if (this.kv == null) { + throw Error("not initialized"); + } + return await this.kv.config(config); + }; + + private updateInventory = asyncThrottle( + async () => { + if (this.isClosed() || this.opts == null || this.opts.noInventory) { + return; + } + await delay(500); + if (this.isClosed() || this.kv == null) { + return; + } + let inv: Inventory | undefined = undefined; + try { + const { account_id, project_id, desc } = this.opts; + const inv = await inventory({ account_id, project_id }); + if (this.isClosed()) { + return; + } + const status = { + type: "kv" as "kv", + name: this.opts.name, + desc, + ...(await this.kv.inventory()), + }; + inv.set(status); + } catch (err) { + if (!process.env.COCALC_TEST_MODE) { + console.log( + `WARNING: unable to update inventory. name='${this.opts.name} -- ${err}'`, + ); + } + } finally { + // @ts-ignore + inv?.close(); + } + }, + INVENTORY_UPDATE_INTERVAL, + { leading: true, trailing: true }, + ); +} + +export const cache = refCache({ + name: "dkv", + createKey: ({ name, account_id, project_id }) => + JSON.stringify({ name, account_id, project_id }), + createObject: async (opts) => { + if (opts.client == null) { + opts = { ...opts, client: await conat() }; + } + const k = new DKV(opts); + await k.init(); + return k; + }, +}); + +export async function dkv(options: DKVOptions): Promise> { + return await cache(options); +} diff --git a/src/packages/conat/sync/dstream.ts b/src/packages/conat/sync/dstream.ts new file mode 100644 index 0000000000..39cf243663 --- /dev/null +++ b/src/packages/conat/sync/dstream.ts @@ -0,0 +1,505 @@ +/* +Eventually Consistent Distributed Message Stream + +DEVELOPMENT: + + +# in node -- note the package directory!! +~/cocalc/src/packages/backend node + +> s = await require("@cocalc/backend/conat/sync").dstream({name:'test'}); +> s = await require("@cocalc/backend/conat/sync").dstream({project_id:cc.current().project_id,name:'foo'});0 + +See the guide for dkv, since it's very similar, especially for use in a browser. +*/ + +import { EventEmitter } from "events"; +import { + CoreStream, + type RawMsg, + type ChangeEvent, + type PublishOptions, +} from "./core-stream"; +import { randomId } from "@cocalc/conat/names"; +import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; +import { isNumericString } from "@cocalc/util/misc"; +import refCache from "@cocalc/util/refcache"; +import { + type Client, + type Headers, + ConatError, +} from "@cocalc/conat/core/client"; +import jsonStableStringify from "json-stable-stringify"; +import type { JSONValue } from "@cocalc/util/types"; +import { Configuration } from "./core-stream"; +import { conat } from "@cocalc/conat/client"; +import { delay, map as awaitMap } from "awaiting"; +import { asyncThrottle, until } from "@cocalc/util/async-utils"; +import { + inventory, + type Inventory, + INVENTORY_UPDATE_INTERVAL, +} from "./inventory"; +import { getLogger } from "@cocalc/conat/client"; + +const logger = getLogger("sync:dstream"); + +export interface DStreamOptions { + // what it's called by us + name: string; + account_id?: string; + project_id?: string; + config?: Partial; + // only load historic messages starting at the given seq number. + start_seq?: number; + desc?: JSONValue; + + client?: Client; + noAutosave?: boolean; + ephemeral?: boolean; + + noCache?: boolean; + noInventory?: boolean; +} + +export class DStream extends EventEmitter { + public readonly name: string; + private stream: CoreStream; + private messages: T[]; + private raw: RawMsg[]; + private noAutosave: boolean; + // TODO: using Map for these will be better because we use .length a bunch, which is O(n) instead of O(1). + private local: { [id: string]: T } = {}; + private publishOptions: { + [id: string]: { headers?: Headers }; + } = {}; + private saved: { [seq: number]: T } = {}; + private opts: DStreamOptions; + + constructor(opts: DStreamOptions) { + super(); + logger.debug("constructor", opts.name); + if (opts.client == null) { + throw Error("client must be specified"); + } + this.opts = opts; + this.noAutosave = !!opts.noAutosave; + this.name = opts.name; + this.stream = new CoreStream(opts); + this.messages = this.stream.messages; + this.raw = this.stream.raw; + return new Proxy(this, { + get(target, prop) { + return typeof prop == "string" && isNumericString(prop) + ? target.get(parseInt(prop)) + : target[String(prop)]; + }, + }); + } + + private initialized = false; + init = async () => { + if (this.initialized) { + throw Error("init can only be called once"); + } + this.initialized = true; + if (this.isClosed()) { + throw Error("closed"); + } + this.stream.on("change", this.handleChange); + this.stream.on("reset", () => { + this.local = {}; + this.saved = {}; + }); + await this.stream.init(); + this.emit("connected"); + }; + + private handleChange = ({ mesg, raw, msgID }: ChangeEvent) => { + if (raw?.seq !== undefined) { + delete this.saved[raw.seq]; + } + if (mesg === undefined) { + return; + } + if (msgID) { + // this is critical with core-stream.ts, since otherwise there is a moment + // when the same message is in both this.local *and* this.messages, and you'll + // see it doubled in this.getAll(). + delete this.local[msgID]; + } + this.emit("change", mesg, raw?.seq); + if (this.isStable()) { + this.emit("stable"); + } + }; + + isStable = () => { + for (const _ in this.saved) { + return false; + } + for (const _ in this.local) { + return false; + } + return true; + }; + + isClosed = () => { + return this.stream == null; + }; + + close = () => { + if (this.isClosed()) { + return; + } + logger.debug("close", this.name); + const stream = this.stream; + stream.removeListener("change", this.handleChange); + // @ts-ignore + delete this.stream; + stream.close(); + this.emit("closed"); + this.removeAllListeners(); + // @ts-ignore + delete this.local; + // @ts-ignore + delete this.messages; + // @ts-ignore + delete this.raw; + // @ts-ignore + delete this.opts; + }; + + get = (n?): T | T[] => { + if (this.isClosed()) { + throw Error("closed"); + } + if (n == null) { + return this.getAll(); + } else { + if (n < this.messages.length) { + return this.messages[n]; + } + const v = Object.keys(this.saved); + if (n < v.length + this.messages.length) { + return this.saved[n - this.messages.length]; + } + return Object.values(this.local)[n - this.messages.length - v.length]; + } + }; + + getAll = (): T[] => { + if (this.isClosed()) { + throw Error("closed"); + } + return [ + ...this.messages, + ...Object.values(this.saved), + ...Object.values(this.local), + ]; + }; + + // sequence number of n-th message + seq = (n: number): number | undefined => { + if (n < this.raw.length) { + return this.raw[n].seq; + } + const v = Object.keys(this.saved); + if (n < v.length + this.raw.length) { + return parseInt(v[n - this.raw.length]); + } + }; + + time = (n: number): Date | undefined => { + if (this.isClosed()) { + throw Error("not initialized"); + } + return this.stream.time(n); + }; + + // all server assigned times of messages in the stream. + times = (): (Date | undefined)[] => { + if (this.isClosed()) { + throw Error("not initialized"); + } + return this.stream.times(); + }; + + get length(): number { + return ( + this.messages.length + + Object.keys(this.saved).length + + Object.keys(this.local).length + ); + } + + publish = ( + mesg: T, + // NOTE: if you call this.headers(n) it is NOT visible until + // the publish is confirmed. This could be changed with more work if it matters. + options?: { headers?: Headers; ttl?: number }, + ): void => { + const id = randomId(); + this.local[id] = mesg; + if (options != null) { + this.publishOptions[id] = options; + } + if (!this.noAutosave) { + this.save(); + } + this.updateInventory(); + }; + + headers = (n) => { + if (this.isClosed()) { + throw Error("closed"); + } + return this.stream.headers(n); + }; + + push = (...args: T[]) => { + if (this.isClosed()) { + throw Error("closed"); + } + for (const mesg of args) { + this.publish(mesg); + } + }; + + hasUnsavedChanges = (): boolean => { + if (this.isClosed()) { + return false; + } + return Object.keys(this.local).length > 0; + }; + + unsavedChanges = (): T[] => { + return Object.values(this.local); + }; + + save = reuseInFlight(async () => { + await until( + async () => { + if (this.isClosed()) { + return true; + } + try { + await this.attemptToSave(); + //console.log("successfully saved"); + } catch (err) { + if (!process.env.COCALC_TEST_MODE) { + console.log( + `WARNING: stream attemptToSave failed - ${err}`, + this.name, + ); + } + } + return !this.hasUnsavedChanges(); + }, + { start: 150, decay: 1.3, max: 10000 }, + ); + }); + + private attemptToSave = async () => { + if (true) { + await this.attemptToSaveBatch(); + } else { + await this.attemptToSaveParallel(); + } + }; + + private attemptToSaveBatch = reuseInFlight(async () => { + if (this.isClosed()) { + throw Error("closed"); + } + const v: { mesg: T; options: PublishOptions }[] = []; + const ids = Object.keys(this.local); + for (const id of ids) { + const mesg = this.local[id]; + const options = { + ...this.publishOptions[id], + msgID: id, + }; + v.push({ mesg, options }); + } + const w: ( + | { seq: number; time: number; error?: undefined } + | { error: string; code?: any } + )[] = await this.stream.publishMany(v); + + if (this.isClosed()) { + return; + } + + let errors = false; + for (let i = 0; i < w.length; i++) { + const id = ids[i]; + if (w[i].error) { + const x = w[i] as { error: string; code?: any }; + if (x.code == "reject") { + delete this.local[id]; + const err = new ConatError(x.error, { code: x.code }); + // err has mesg and subject set. + this.emit("reject", { err, mesg: v[i].mesg }); + } + if (!process.env.COCALC_TEST_MODE) { + console.warn( + `WARNING -- error saving dstream '${this.name}' -- ${w[i].error}`, + ); + } + errors = true; + continue; + } + const { seq } = w[i] as { seq: number }; + if ((this.raw[this.raw.length - 1]?.seq ?? -1) < seq) { + // it still isn't in this.raw + this.saved[seq] = v[i].mesg; + } + delete this.local[id]; + delete this.publishOptions[id]; + } + if (errors) { + throw Error(`there were errors saving dstream '${this.name}'`); + } + }); + + // non-batched version + private attemptToSaveParallel = reuseInFlight(async () => { + const f = async (id) => { + if (this.isClosed()) { + throw Error("closed"); + } + const mesg = this.local[id]; + try { + // @ts-ignore + const { seq } = await this.stream.publish(mesg, { + ...this.publishOptions[id], + msgID: id, + }); + if (this.isClosed()) { + return; + } + if ((this.raw[this.raw.length - 1]?.seq ?? -1) < seq) { + // it still isn't in this.raw + this.saved[seq] = mesg; + } + delete this.local[id]; + delete this.publishOptions[id]; + } catch (err) { + if (err.code == "reject") { + delete this.local[id]; + // err has mesg and subject set. + this.emit("reject", { err, mesg }); + } else { + if (!process.env.COCALC_TEST_MODE) { + console.warn( + `WARNING: problem saving dstream ${this.name} -- ${err}`, + ); + } + } + } + if (this.isStable()) { + this.emit("stable"); + } + }; + // NOTE: ES6 spec guarantees "String keys are returned in the order + // in which they were added to the object." + const ids = Object.keys(this.local); + const MAX_PARALLEL = 50; + await awaitMap(ids, MAX_PARALLEL, f); + }); + + // load older messages starting at start_seq + load = async (opts: { start_seq: number }) => { + if (this.isClosed()) { + throw Error("closed"); + } + await this.stream.load(opts); + }; + + // this is not synchronous -- it makes sure everything is saved out, + // then delete the persistent stream + // NOTE: for ephemeral streams, other clients will NOT see the result of a purge (unless they reconnect). + delete = async (opts?) => { + await this.save(); + if (this.isClosed()) { + throw Error("closed"); + } + return await this.stream.delete(opts); + }; + + get start_seq(): number | undefined { + return this.stream?.start_seq; + } + + // get or set config + config = async ( + config: Partial = {}, + ): Promise => { + if (this.isClosed()) { + throw Error("closed"); + } + return await this.stream.config(config); + }; + + private updateInventory = asyncThrottle( + async () => { + if (this.isClosed() || this.opts == null || this.opts.noInventory) { + return; + } + await delay(500); + if (this.isClosed()) { + return; + } + let inv: Inventory | undefined = undefined; + try { + const { account_id, project_id, desc } = this.opts; + const inv = await inventory({ account_id, project_id }); + if (this.isClosed()) { + return; + } + const status = { + type: "stream" as "stream", + name: this.opts.name, + desc, + ...(await this.stream.inventory()), + }; + inv.set(status); + } catch (err) { + if (!process.env.COCALC_TEST_MODE) { + console.log( + `WARNING: unable to update inventory. name='${this.opts.name} -- ${err}'`, + ); + } + } finally { + // @ts-ignore + inv?.close(); + } + }, + INVENTORY_UPDATE_INTERVAL, + { leading: true, trailing: true }, + ); +} + +export const cache = refCache({ + name: "dstream", + createKey: (options: DStreamOptions) => { + if (!options.name) { + throw Error("name must be specified"); + } + const { name, account_id, project_id } = options; + return jsonStableStringify({ name, account_id, project_id })!; + }, + createObject: async (options: DStreamOptions) => { + if (options.client == null) { + options = { ...options, client: await conat() }; + } + const dstream = new DStream(options); + await dstream.init(); + return dstream; + }, +}); + +export async function dstream(options: DStreamOptions): Promise> { + return await cache(options); +} diff --git a/src/packages/conat/sync/inventory.ts b/src/packages/conat/sync/inventory.ts new file mode 100644 index 0000000000..380983b5c9 --- /dev/null +++ b/src/packages/conat/sync/inventory.ts @@ -0,0 +1,343 @@ +/* +Inventory of all streams and key:value stores in a specific project or account. + +DEVELOPMENT: + +i = await require('@cocalc/backend/conat/sync').inventory({project_id:'00847397-d6a8-4cb0-96a8-6ef64ac3e6cf'}) + +i.ls() + +*/ + +import { dkv, type DKV } from "./dkv"; +import { dstream, type DStream } from "./dstream"; +import { dko, type DKO } from "./dko"; +import getTime from "@cocalc/conat/time"; +import refCache from "@cocalc/util/refcache"; +import type { JSONValue } from "@cocalc/util/types"; +import { + human_readable_size as humanReadableSize, + trunc_middle, +} from "@cocalc/util/misc"; +import { DKO_PREFIX } from "./dko"; +import { waitUntilTimeAvailable } from "@cocalc/conat/time"; +import { + type Configuration, + type PartialInventory, +} from "@cocalc/conat/persist/storage"; +import { AsciiTable3 } from "ascii-table3"; + +export const INVENTORY_UPDATE_INTERVAL = 30000; +export const THROTTLE_MS = 10000; +export const INVENTORY_NAME = "CoCalc-Inventory"; + +type Sort = + | "last" + | "created" + | "count" + | "bytes" + | "name" + | "type" + | "-last" + | "-created" + | "-count" + | "-bytes" + | "-name" + | "-type"; + +interface Location { + account_id?: string; + project_id?: string; +} + +type StoreType = "stream" | "kv"; + +export interface InventoryItem extends PartialInventory { + // when it was created + created: number; + // last time this stream was updated + last: number; + // optional description, which can be anything + desc?: JSONValue; +} + +interface FullItem extends InventoryItem { + type: StoreType; + name: string; +} + +export class Inventory { + public location: Location; + private dkv?: DKV; + + constructor(location: { account_id?: string; project_id?: string }) { + this.location = location; + } + + init = async () => { + this.dkv = await dkv({ + name: INVENTORY_NAME, + ...this.location, + }); + await waitUntilTimeAvailable(); + }; + + // Set but with NO LIMITS and no MERGE conflict algorithm. Use with care! + set = ({ + type, + name, + bytes, + count, + desc, + limits, + seq, + }: { + type: StoreType; + name: string; + bytes: number; + count: number; + limits: Partial; + desc?: JSONValue; + seq: number; + }) => { + if (this.dkv == null) { + throw Error("not initialized"); + } + const last = getTime(); + const key = this.encodeKey({ name, type }); + const cur = this.dkv.get(key); + const created = cur?.created ?? last; + desc = desc ?? cur?.desc; + this.dkv.set(key, { + desc, + last, + created, + bytes, + count, + limits, + seq, + }); + }; + + private encodeKey = ({ name, type }) => JSON.stringify({ name, type }); + + private decodeKey = (key) => JSON.parse(key); + + delete = ({ name, type }: { name: string; type: StoreType }) => { + if (this.dkv == null) { + throw Error("not initialized"); + } + this.dkv.delete(this.encodeKey({ name, type })); + }; + + get = ( + x: { name: string; type: StoreType } | string, + ): (InventoryItem & { type: StoreType; name: string }) | undefined => { + if (this.dkv == null) { + throw Error("not initialized"); + } + let cur; + let name, type; + if (typeof x == "string") { + // just the name -- we infer/guess the type + name = x; + type = "kv"; + cur = this.dkv.get(this.encodeKey({ name, type })); + if (cur == null) { + type = "stream"; + cur = this.dkv.get(this.encodeKey({ name, type })); + } + } else { + name = x.name; + cur = this.dkv.get(this.encodeKey(x)); + } + if (cur == null) { + return; + } + return { ...cur, type, name }; + }; + + getStores = async ({ + filter, + sort = "-last", + }: { filter?: string; sort?: Sort } = {}): Promise< + (DKV | DStream | DKO)[] + > => { + const v: (DKV | DStream | DKO)[] = []; + const all = this.getAll({ filter }); + for (const key of this.sortedKeys(all, sort)) { + const x = all[key]; + const { desc, name, type } = x; + if (type == "kv") { + if (name.startsWith(DKO_PREFIX)) { + v.push(await dko({ name, ...this.location, desc })); + } else { + v.push(await dkv({ name, ...this.location, desc })); + } + } else if (type == "stream") { + v.push(await dstream({ name, ...this.location, desc })); + } else { + throw Error(`unknown store type '${type}'`); + } + } + return v; + }; + + getAll = ({ filter }: { filter?: string } = {}): FullItem[] => { + if (this.dkv == null) { + throw Error("not initialized"); + } + const all = this.dkv.getAll(); + if (filter) { + filter = filter.toLowerCase(); + } + const v: FullItem[] = []; + for (const key of Object.keys(all)) { + const { name, type } = this.decodeKey(key); + if (filter) { + const { desc } = all[key]; + const s = `${desc ? JSON.stringify(desc) : ""} ${name}`.toLowerCase(); + if (!s.includes(filter)) { + continue; + } + } + v.push({ ...all[key], name, type }); + } + return v; + }; + + close = async () => { + await this.dkv?.close(); + delete this.dkv; + }; + + private sortedKeys = (all, sort0: Sort) => { + let reverse: boolean, sort: string; + if (sort0[0] == "-") { + reverse = true; + sort = sort0.slice(1); + } else { + reverse = false; + sort = sort0; + } + // return keys of all, sorted as specified + const x: { k: string; v: any }[] = []; + for (const k in all) { + x.push({ k, v: { ...all[k], ...this.decodeKey(k) } }); + } + x.sort((a, b) => { + const a0 = a.v[sort]; + const b0 = b.v[sort]; + if (a0 < b0) { + return -1; + } + if (a0 > b0) { + return 1; + } + return 0; + }); + const y = x.map(({ k }) => k); + if (reverse) { + y.reverse(); + } + return y; + }; + + ls = ({ + log = console.log, + filter, + noTrunc, + path: path0, + sort = "last", + noHelp, + }: { + log?: Function; + filter?: string; + noTrunc?: boolean; + path?: string; + sort?: Sort; + noHelp?: boolean; + } = {}) => { + if (this.dkv == null) { + throw Error("not initialized"); + } + const all = this.dkv.getAll(); + if (!noHelp) { + log( + "ls(opts: {filter?: string; noTrunc?: boolean; path?: string; sort?: 'last'|'created'|'count'|'bytes'|'name'|'type'|'-last'|...})", + ); + } + + const rows: any[] = []; + for (const key of this.sortedKeys(all, sort)) { + const { last, created, count, bytes, desc, limits } = all[key]; + if (path0 && desc?.["path"] != path0) { + continue; + } + let { name, type } = this.decodeKey(key); + if (name.startsWith(DKO_PREFIX)) { + type = "kvobject"; + name = name.slice(DKO_PREFIX.length); + } + if (!noTrunc) { + name = trunc_middle(name, 50); + } + if ( + filter && + !`${desc ? JSON.stringify(desc) : ""} ${name}` + .toLowerCase() + .includes(filter.toLowerCase()) + ) { + continue; + } + rows.push([ + type, + name, + dateToString(new Date(created)), + humanReadableSize(bytes), + count, + dateToString(new Date(last)), + desc ? JSON.stringify(desc) : "", + Object.keys(limits).length > 0 ? JSON.stringify(limits) : "--", + ]); + } + + const table = new AsciiTable3( + `Inventory for ${JSON.stringify(this.location)}`, + ) + .setHeading( + "Type", + "Name", + "Created", + "Size", + "Count", + "Last Update", + "Desc", + "Limits", + ) + .addRowMatrix(rows); + table.setStyle("unicode-round"); + if (!noTrunc) { + table.setWidth(7, 50).setWrapped(1); + table.setWidth(8, 30).setWrapped(1); + } + log(table.toString()); + }; +} + +function dateToString(d: Date) { + return d.toISOString().replace("T", " ").replace("Z", "").split(".")[0]; +} + +export const cache = refCache({ + name: "inventory", + createObject: async (loc) => { + const k = new Inventory(loc); + await k.init(); + return k; + }, +}); + +export async function inventory(options: Location = {}): Promise { + return await cache(options); +} diff --git a/src/packages/conat/sync/limits.ts b/src/packages/conat/sync/limits.ts new file mode 100644 index 0000000000..6301bf083f --- /dev/null +++ b/src/packages/conat/sync/limits.ts @@ -0,0 +1,178 @@ +import type { RawMsg } from "./core-stream"; + +export const ENFORCE_LIMITS_THROTTLE_MS = process.env.COCALC_TEST_MODE + ? 100 + : 45000; + +class PublishRejectError extends Error { + code: string; + mesg: any; + subject?: string; + limit?: string; +} + +export interface FilteredStreamLimitOptions { + // How many messages may be in a Stream, oldest messages will be removed + // if the Stream exceeds this size. -1 for unlimited. + max_msgs: number; + // Maximum age of any message in the stream matching the filter, + // expressed in milliseconds. 0 for unlimited. + // **Note that max_age is in milliseconds.** + max_age: number; + // How big the Stream may be, when the combined stream size matching the filter + // exceeds this old messages are removed. -1 for unlimited. + // This is enforced only on write, so if you change it, it only applies + // to future messages. + max_bytes: number; + // The largest message that will be accepted by the Stream. -1 for unlimited. + max_msg_size: number; + + // Attempting to publish a message that causes this to be exceeded + // throws an exception instead. -1 (or 0) for unlimited + // For dstream, the messages are explicitly rejected and the client + // gets a "reject" event emitted. E.g., the terminal running in the project + // writes [...] when it gets these rejects, indicating that data was + // dropped. + max_bytes_per_second: number; + max_msgs_per_second: number; +} + +export interface KVLimits { + // How many keys may be in the KV store. Oldest keys will be removed + // if the key-value store exceeds this size. -1 for unlimited. + max_msgs: number; + + // Maximum age of any key, expressed in milliseconds. 0 for unlimited. + // Age is updated whenever value of the key is changed. + max_age: number; + + // The maximum number of bytes to store in this KV, which means + // the total of the bytes used to store everything. Since we store + // the key with each value (to have arbitrary keys), this includes + // the size of the keys. + max_bytes: number; + + // The maximum size of any single value, including the key. + max_msg_size: number; +} + +export function enforceLimits({ + messages, + raw, + limits, +}: { + messages: T[]; + raw: RawMsg[]; + limits: FilteredStreamLimitOptions; +}) { + const { max_msgs, max_age, max_bytes } = limits; + // we check with each defined limit if some old messages + // should be dropped, and if so move limit forward. If + // it is above -1 at the end, we do the drop. + let index = -1; + const setIndex = (i, _limit) => { + // console.log("setIndex", { i, _limit }); + index = Math.max(i, index); + }; + // max_msgs + // console.log({ max_msgs, l: messages.length, messages }); + if (max_msgs > -1 && messages.length > max_msgs) { + // ensure there are at most limits.max_msgs messages + // by deleting the oldest ones up to a specified point. + const i = messages.length - max_msgs; + if (i > 0) { + setIndex(i - 1, "max_msgs"); + } + } + + // max_age + if (max_age > 0) { + // expire messages older than max_age nanoseconds + const recent = raw[raw.length - 1]; + if (recent != null) { + // to avoid potential clock skew, we define *now* as the time of the most + // recent message. For us, this should be fine, since we only impose limits + // when writing new messages, and none of these limits are guaranteed. + const now = recent.timestamp; + if (now) { + const cutoff = now - max_age; + for (let i = raw.length - 1; i >= 0; i--) { + const t = raw[i].timestamp; + if (t < cutoff) { + // it just went over the limit. Everything before + // and including the i-th message must be deleted. + setIndex(i, "max_age"); + break; + } + } + } + } + } + + // max_bytes + if (max_bytes >= 0) { + let t = 0; + for (let i = raw.length - 1; i >= 0; i--) { + t += raw[i].data.length; + if (t > max_bytes) { + // it just went over the limit. Everything before + // and including the i-th message must be deleted. + setIndex(i, "max_bytes"); + break; + } + } + } + + return index; +} + +export function enforceRateLimits({ + limits, + bytesSent, + subject, + bytes, +}: { + limits: { max_bytes_per_second: number; max_msgs_per_second: number }; + bytesSent: { [time: number]: number }; + subject?: string; + bytes; +}) { + const now = Date.now(); + if (!(limits.max_bytes_per_second > 0) && !(limits.max_msgs_per_second > 0)) { + return; + } + + const cutoff = now - 1000; + let totalBytes = 0, + msgs = 0; + for (const t in bytesSent) { + if (parseInt(t) < cutoff) { + delete bytesSent[t]; + } else { + totalBytes += bytesSent[t]; + msgs += 1; + } + } + if ( + limits.max_bytes_per_second > 0 && + totalBytes + bytes > limits.max_bytes_per_second + ) { + const err = new PublishRejectError( + `bytes per second limit of ${limits.max_bytes_per_second} exceeded`, + ); + err.code = "REJECT"; + err.subject = subject; + err.limit = "max_bytes_per_second"; + throw err; + } + if (limits.max_msgs_per_second > 0 && msgs > limits.max_msgs_per_second) { + const err = new PublishRejectError( + `messages per second limit of ${limits.max_msgs_per_second} exceeded`, + ); + err.code = "REJECT"; + err.subject = subject; + err.limit = "max_msgs_per_second"; + throw err; + } + bytesSent[now] = bytes; +} diff --git a/src/packages/nats/sync/open-files.ts b/src/packages/conat/sync/open-files.ts similarity index 58% rename from src/packages/nats/sync/open-files.ts rename to src/packages/conat/sync/open-files.ts index 202b040fef..b82afd8578 100644 --- a/src/packages/nats/sync/open-files.ts +++ b/src/packages/conat/sync/open-files.ts @@ -1,13 +1,28 @@ /* -NATS Kv associated to a project to keep track of open files. +Keep track of open files. + +We use the "dko" distributed key:value store because of the potential of merge +conflicts, e.g,. one client changes the compute server id and another changes +whether a file is deleted. By using dko, only the field that changed is sync'd +out, so we get last-write-wins on the level of fields. + +WARNINGS: +An old version use dkv with merge conflict resolution, but with multiple clients +and the project, feedback loops or something happened and it would start getting +slow -- basically, merge conflicts could take a few seconds to resolve, which would +make opening a file start to be slow. Instead we use DKO data type, where fields +are treated separately atomically by the storage system. A *subtle issue* is +that when you set an object, this is NOT treated atomically. E.g., if you +set 2 fields in a set operation, then 2 distinct changes are emitted as the +two fields get set. DEVELOPMENT: -Change to packages/backend, since packages/nats doesn't have a way to connect: +Change to packages/backend, since packages/conat doesn't have a way to connect: ~/cocalc/src/packages/backend$ node -> z = await require('@cocalc/backend/nats/sync').openFiles({project_id:cc.current().project_id}) +> z = await require('@cocalc/backend/conat/sync').openFiles({project_id:cc.current().project_id}) > z.touch({path:'a.txt'}) > z.get({path:'a.txt'}) { open: true, count: 1, time:2025-02-09T16:37:20.713Z } @@ -24,20 +39,19 @@ Change to packages/backend, since packages/nats doesn't have a way to connect: Frontend Dev in browser: -z = await cc.client.nats_client.openFiles({project_id:cc.current().project_id)) +z = await cc.client.conat_client.openFiles({project_id:cc.current().project_id)) z.getAll() } */ -import { type State } from "@cocalc/nats/types"; -import { dkv, type DKV } from "@cocalc/nats/sync/dkv"; +import { type State } from "@cocalc/conat/types"; +import { dko, type DKO } from "@cocalc/conat/sync/dko"; import { EventEmitter } from "events"; -import getTime, { getSkew } from "@cocalc/nats/time"; -import { getEnv } from "@cocalc/nats/client"; +import getTime, { getSkew } from "@cocalc/conat/time"; // info about interest in open files (and also what was explicitly deleted) older // than this is automatically purged. -const MAX_AGE_MS = 7 * (1000 * 60 * 60 * 24); +const MAX_AGE_MS = 1000 * 60 * 60 * 24; interface Deleted { // what deleted state is @@ -53,8 +67,6 @@ interface Backend { time: number; } -// IMPORTANT: if you add/change any fields below, be sure to update -// the merge conflict function! export interface KVEntry { // a web browser has the file open at this point in time (in ms) time?: number; @@ -68,22 +80,14 @@ export interface KVEntry { // of merge conflict we can do something sensible. deleted?: Deleted; - // if file is actively opened on a compute server, then it sets + // if file is actively opened on a compute server/project, then it sets // this entry. Right when it closes the file, it clears this. // If it gets killed/broken and doesn't have a chance to clear it, then // backend.time can be used to decide this isn't valid. backend?: Backend; -} -function resolveMergeConflict(local: KVEntry, remote: KVEntry): KVEntry { - const time = mergeTime(remote?.time, local?.time); - const deleted = mergeDeleted(remote?.deleted, local?.deleted); - const backend = mergeBackend(remote?.backend, local?.backend); - return { - time, - deleted, - backend, - }; + // optional information + doctype?; } export interface Entry extends KVEntry { @@ -91,43 +95,6 @@ export interface Entry extends KVEntry { path: string; } -function mergeTime( - a: number | undefined, - b: number | undefined, -): number | undefined { - // time of interest should clearly always be the largest known value so far. - if (a == null && b == null) { - return undefined; - } - return Math.max(a ?? 0, b ?? 0); -} - -function mergeDeleted(a: Deleted | undefined, b: Deleted | undefined) { - if (a == null) { - return b; - } - if (b == null) { - return a; - } - // now both a and b are not null, so some merge is needed: we - // use last write wins. - return a.time >= b.time ? a : b; -} - -function mergeBackend(a: Backend | undefined, b: Backend | undefined) { - if (a == null) { - return b; - } - if (b == null) { - return a; - } - // now both a and b are not null, so some merge is needed: we - // use last write wins. - // NOTE: This should likely not happen or only happen for a moment and - // would be worrisome, but quickly sort itself out. - return a.time >= b.time ? a : b; -} - interface Options { project_id: string; noAutosave?: boolean; @@ -144,7 +111,7 @@ export class OpenFiles extends EventEmitter { private project_id: string; private noCache?: boolean; private noAutosave?: boolean; - private dkv?: DKV; + private kv?: DKO; public state: "disconnected" | "connected" | "closed" = "disconnected"; constructor({ project_id, noAutosave, noCache }: Options) { @@ -162,102 +129,115 @@ export class OpenFiles extends EventEmitter { this.emit(state); }; + private initialized = false; init = async () => { - const d = await dkv({ + if (this.initialized) { + throw Error("init can only be called once"); + } + this.initialized = true; + const d = await dko({ name: "open-files", project_id: this.project_id, - env: await getEnv(), - limits: { + config: { max_age: MAX_AGE_MS, }, noAutosave: this.noAutosave, noCache: this.noCache, - merge: ({ local, remote }) => resolveMergeConflict(local, remote), - }); - this.dkv = d; - d.on("change", ({ key: path }) => { - const entry = this.get(path); - if (entry != null) { - // not deleted and timestamp is set: - this.emit("change", entry as Entry); - } + noInventory: true, }); + this.kv = d; + d.on("change", this.handleChange); // ensure clock is synchronized await getSkew(); this.setState("connected"); }; + private handleChange = ({ key: path }) => { + const entry = this.get(path); + if (entry != null) { + // not deleted and timestamp is set: + this.emit("change", entry as Entry); + } + }; + close = () => { - if (this.dkv == null) { + if (this.kv == null) { return; } this.setState("closed"); this.removeAllListeners(); - this.dkv.close(); - delete this.dkv; + this.kv.removeListener("change", this.handleChange); + this.kv.close(); + delete this.kv; // @ts-ignore delete this.project_id; }; - private getDkv = () => { - const { dkv } = this; - if (dkv == null) { + private getKv = () => { + const { kv } = this; + if (kv == null) { throw Error("closed"); } - return dkv; + return kv; }; private set = (path, entry: KVEntry) => { - this.getDkv().set(path, entry); + this.getKv().set(path, entry); }; // When a client has a file open, they should periodically // touch it to indicate that it is open. // updates timestamp and ensures open=true. - touch = (path: string) => { + touch = (path: string, doctype?) => { if (!path) { throw Error("path must be specified"); } - const dkv = this.getDkv(); - // n = sequence number to make sure a write happens, which updates - // server assigned timestamp. - const cur = dkv.get(path); + const kv = this.getKv(); + const cur = kv.get(path); const time = getTime(); - this.set(path, { - ...cur, - time, - }); + if (doctype) { + this.set(path, { + ...cur, + time, + doctype, + }); + } else { + this.set(path, { + ...cur, + time, + }); + } }; setError = (path: string, err?: any) => { - const dkv = this.getDkv(); + const kv = this.getKv(); if (!err) { - const current = { ...dkv.get(path) }; + const current = { ...kv.get(path) }; delete current.error; this.set(path, current); } else { - const current = { ...dkv.get(path) }; + const current = { ...kv.get(path) }; current.error = { time: Date.now(), error: `${err}` }; this.set(path, current); } }; setDeleted = (path: string) => { - const dkv = this.getDkv(); + const kv = this.getKv(); this.set(path, { - ...dkv.get(path), + ...kv.get(path), deleted: { deleted: true, time: getTime() }, }); }; isDeleted = (path: string) => { - return !!this.getDkv().get(path)?.deleted?.deleted; + return !!this.getKv().get(path)?.deleted?.deleted; }; setNotDeleted = (path: string) => { - const dkv = this.getDkv(); + const kv = this.getKv(); this.set(path, { - ...dkv.get(path), + ...kv.get(path), deleted: { deleted: false, time: getTime() }, }); }; @@ -266,23 +246,23 @@ export class OpenFiles extends EventEmitter { // This should be called by that backend periodically // when it has the file opened. setBackend = (path: string, id: number) => { - const dkv = this.getDkv(); + const kv = this.getKv(); this.set(path, { - ...dkv.get(path), + ...kv.get(path), backend: { id, time: getTime() }, }); }; // get current backend that has file opened. getBackend = (path: string): Backend | undefined => { - return this.getDkv().get(path)?.backend; + return this.getKv().get(path)?.backend; }; // ONLY if backend for path is currently set to id, then clear // the backend field. setNotBackend = (path: string, id: number) => { - const dkv = this.getDkv(); - const cur = { ...dkv.get(path) }; + const kv = this.getKv(); + const cur = { ...kv.get(path) }; if (cur?.backend?.id == id) { delete cur.backend; this.set(path, cur); @@ -290,14 +270,14 @@ export class OpenFiles extends EventEmitter { }; getAll = (): Entry[] => { - const x = this.getDkv().getAll(); + const x = this.getKv().getAll(); return Object.keys(x).map((path) => { return { ...x[path], path }; }); }; get = (path: string): Entry | undefined => { - const x = this.getDkv().get(path); + const x = this.getKv().get(path); if (x == null) { return x; } @@ -305,18 +285,18 @@ export class OpenFiles extends EventEmitter { }; delete = (path) => { - this.getDkv().delete(path); + this.getKv().delete(path); }; clear = () => { - this.getDkv().clear(); + this.getKv().clear(); }; save = async () => { - await this.getDkv().save(); + await this.getKv().save(); }; hasUnsavedChanges = () => { - return this.getDkv().hasUnsavedChanges(); + return this.getKv().hasUnsavedChanges(); }; } diff --git a/src/packages/nats/sync/pubsub.ts b/src/packages/conat/sync/pubsub.ts similarity index 58% rename from src/packages/nats/sync/pubsub.ts rename to src/packages/conat/sync/pubsub.ts index cbaeab8baa..329ef81d03 100644 --- a/src/packages/nats/sync/pubsub.ts +++ b/src/packages/conat/sync/pubsub.ts @@ -1,16 +1,17 @@ /* -Use NATS simple pub/sub to share state for something *ephemeral* in a project. +Use Conat simple pub/sub to share state for something very *ephemeral* in a project. + +This is used, e.g., for broadcasting a user's cursors when they are editing a file. */ -import { projectSubject } from "@cocalc/nats/names"; -import { type NatsEnv, State } from "@cocalc/nats/types"; +import { projectSubject } from "@cocalc/conat/names"; +import { State } from "@cocalc/conat/types"; import { EventEmitter } from "events"; -import { isConnectedSync } from "@cocalc/nats/util"; -import { type Subscription } from "@nats-io/nats-core"; +import { type Subscription, getClient, Client } from "@cocalc/conat/core/client"; export class PubSub extends EventEmitter { private subject: string; - private env: NatsEnv; + private client: Client; private sub?: Subscription; private state: State = "disconnected"; @@ -18,15 +19,15 @@ export class PubSub extends EventEmitter { project_id, path, name, - env, + client, }: { project_id: string; name: string; path?: string; - env: NatsEnv; + client?: Client; }) { super(); - this.env = env; + this.client = client ?? getClient(); this.subject = projectSubject({ project_id, path, @@ -52,18 +53,14 @@ export class PubSub extends EventEmitter { }; set = (obj) => { - if (!isConnectedSync()) { - // when disconnected, all state is dropped - return; - } - this.env.nc.publish(this.subject, this.env.jc.encode(obj)); + this.client.publish(this.subject, obj); }; private subscribe = async () => { - this.sub = this.env.nc.subscribe(this.subject); + this.sub = await this.client.subscribe(this.subject); this.setState("connected"); for await (const mesg of this.sub) { - this.emit("change", this.env.jc.decode(mesg.data)); + this.emit("change", mesg.data); } }; } diff --git a/src/packages/nats/sync/syncdoc-info.ts b/src/packages/conat/sync/syncdoc-info.ts similarity index 89% rename from src/packages/nats/sync/syncdoc-info.ts rename to src/packages/conat/sync/syncdoc-info.ts index 042c4d4ee9..825f7de11d 100644 --- a/src/packages/nats/sync/syncdoc-info.ts +++ b/src/packages/conat/sync/syncdoc-info.ts @@ -5,14 +5,14 @@ export async function getSyncDocType({ project_id, path, }): Promise<{ type: "db" | "string"; opts?: any }> { - // instead of just "querying the db" (i.e., nats in this case), + // instead of just "querying the db" (i.e., conat in this case), // we create the synctable. This avoids race conditions, since we // can wait until data is written, and also abstracts away the // internal structure. let syncdocs; try { const string_id = client_db.sha1(project_id, path); - syncdocs = await client.synctable_nats( + syncdocs = await client.synctable_conat( { syncstrings: [{ project_id, path, string_id, doctype: null }] }, { stream: false, @@ -22,7 +22,7 @@ export async function getSyncDocType({ ); let s = syncdocs.get_one(); if (s?.doctype == null) { - // wait until there is a syncstring and its doctype is set: + // wait until there is a syncstring and its doctype is set (this should be done by the frontend) await syncdocs.wait(() => { s = syncdocs.get_one(); return s?.doctype != null; diff --git a/src/packages/nats/sync/synctable-kv.ts b/src/packages/conat/sync/synctable-kv.ts similarity index 83% rename from src/packages/nats/sync/synctable-kv.ts rename to src/packages/conat/sync/synctable-kv.ts index e7ae924309..f01e937921 100644 --- a/src/packages/nats/sync/synctable-kv.ts +++ b/src/packages/conat/sync/synctable-kv.ts @@ -7,16 +7,17 @@ import { keys } from "lodash"; import { client_db } from "@cocalc/util/db-schema/client-db"; -import type { NatsEnv, State } from "@cocalc/nats/types"; +import type { State } from "@cocalc/conat/types"; +import type { Client } from "@cocalc/conat/core/client"; import { EventEmitter } from "events"; import { dkv as createDkv, type DKV } from "./dkv"; import { dko as createDko, type DKO } from "./dko"; import jsonStableStringify from "json-stable-stringify"; -import { toKey } from "@cocalc/nats/util"; +import { toKey } from "@cocalc/conat/util"; import { wait } from "@cocalc/util/async-wait"; import { fromJS, Map } from "immutable"; -import { type KVLimits } from "./general-kv"; import type { JSONValue } from "@cocalc/util/types"; +import type { Configuration } from "@cocalc/conat/sync/core-stream"; export class SyncTableKV extends EventEmitter { public readonly table; @@ -27,42 +28,42 @@ export class SyncTableKV extends EventEmitter { private account_id?: string; private state: State = "disconnected"; private dkv?: DKV | DKO; - private env; + private client: Client; private getHook: Function; - private limits?: Partial; + private config?: Partial; private desc?: JSONValue; - private noInventory?: boolean; + private ephemeral?: boolean; constructor({ query, - env, + client, account_id, project_id, atomic, immutable, - limits, + config, desc, - noInventory, + ephemeral, }: { query; - env: NatsEnv; + client: Client; account_id?: string; project_id?: string; atomic?: boolean; immutable?: boolean; - limits?: Partial; + config?: Partial; desc?: JSONValue; - noInventory?: boolean; + ephemeral?: boolean; }) { super(); - this.noInventory = noInventory; - this.setMaxListeners(100); + this.setMaxListeners(1000); this.atomic = !!atomic; this.getHook = immutable ? fromJS : (x) => x; this.query = query; - this.limits = limits; - this.env = env; + this.config = config; + this.client = client; this.desc = desc; + this.ephemeral = ephemeral; this.table = keys(query)[0]; if (query[this.table][0].string_id && query[this.table][0].project_id) { this.project_id = query[this.table][0].project_id; @@ -84,7 +85,7 @@ export class SyncTableKV extends EventEmitter { // WARNING: be *VERY* careful before changing how the name is // derived from the query, since if you change this all the current - // data in NATS that caches the changefeeds is basically lost + // data in conat that caches the changefeeds is basically lost // and users MUST refresh their browsers (and maybe projects restart?) // to get new changefeeds, since they are watching something given // by this name. I.e., this name shouldn't ever be changed. @@ -93,9 +94,9 @@ export class SyncTableKV extends EventEmitter { // A big choice here is the full name or just something short like the // sha1 hash, but I've chosen the full name, since then it is always easy // to know what the query was, i.e., use base64 decoding then you - // have the query. It's less efficient though since the NATS subjects + // have the query. It's less efficient though since the conat subjects // can be long, depending on the query. - // This way if we are just watching general NATS traffic and see something + // This way if we are just watching general conat traffic and see something // suspicious, even if we have no idea initially where it came from, // we can easily see by decoding it. // Including even the fields with no values distinguishes different @@ -116,25 +117,29 @@ export class SyncTableKV extends EventEmitter { init = async () => { const name = this.getName(); + console.log("initializaing a synctable-kv", { + name, + spec: this.query[this.table][0], + }); if (this.atomic) { this.dkv = await createDkv({ + client: this.client, name, account_id: this.account_id, project_id: this.project_id, - env: this.env, - limits: this.limits, + config: this.config, desc: this.desc, - noInventory: this.noInventory, + ephemeral: this.ephemeral, }); } else { this.dkv = await createDko({ + client: this.client, name, account_id: this.account_id, project_id: this.project_id, - env: this.env, - limits: this.limits, + config: this.config, desc: this.desc, - noInventory: this.noInventory, + ephemeral: this.ephemeral, }); } // For some reason this one line confuses typescript and break building the compute server package (nothing else similar happens). @@ -151,7 +156,7 @@ export class SyncTableKV extends EventEmitter { } } // change api was to emit array of keys. - // We also use this packages/sync/table/changefeed-nats.ts which needs the value, + // We also use this packages/sync/table/changefeed-conat.ts which needs the value, // so we emit that object second. this.emit("change", [x.key], x); }); @@ -214,7 +219,7 @@ export class SyncTableKV extends EventEmitter { await this.dkv?.close(); delete this.dkv; // @ts-ignore - delete this.env; + delete this.client; }; public async wait(until: Function, timeout: number = 30): Promise { diff --git a/src/packages/nats/sync/synctable-stream.ts b/src/packages/conat/sync/synctable-stream.ts similarity index 87% rename from src/packages/nats/sync/synctable-stream.ts rename to src/packages/conat/sync/synctable-stream.ts index a810af96fb..36b1b3e27e 100644 --- a/src/packages/nats/sync/synctable-stream.ts +++ b/src/packages/conat/sync/synctable-stream.ts @@ -1,11 +1,12 @@ /* -Nats implementation of the idea of a "SyncTable", but +Conat implementation of the idea of a "SyncTable", but for streaming data. **This is ONLY for the scope of patches in a single project and IS NOT USED IN ANY WAY WITH POSTGRESQL.** -It uses a NATS stream to store the elements in a well defined order. +It uses a conat persistent stream to store the elements +in a well defined order. */ import jsonStableStringify from "json-stable-stringify"; @@ -13,10 +14,10 @@ import { keys } from "lodash"; import { cmp_Date, is_array, isValidUUID } from "@cocalc/util/misc"; import { client_db } from "@cocalc/util/db-schema/client-db"; import { EventEmitter } from "events"; -import { type NatsEnv } from "@cocalc/nats/types"; import { dstream, DStream } from "./dstream"; import { fromJS, Map } from "immutable"; -import { type FilteredStreamLimitOptions } from "./stream"; +import type { Configuration } from "@cocalc/conat/sync/core-stream"; +import type { Client } from "@cocalc/conat/core/client"; export type State = "disconnected" | "connected" | "closed"; @@ -38,38 +39,42 @@ export class SyncTableStream extends EventEmitter { private string_id: string; private data: any = {}; private state: State = "disconnected"; - private env; private dstream?: DStream; + private client: Client; private getHook: Function; - private limits?: Partial; + private config?: Partial; private start_seq?: number; private noInventory?: boolean; + private ephemeral?: boolean; constructor({ query, - env, + client, account_id: _account_id, project_id, immutable, - limits, + config, start_seq, noInventory, + ephemeral, }: { query; - env: NatsEnv; + client: Client; account_id?: string; project_id?: string; immutable?: boolean; - limits?: Partial; + config?: Partial; start_seq?: number; noInventory?: boolean; + ephemeral?: boolean; }) { super(); + this.client = client; this.noInventory = noInventory; - this.setMaxListeners(100); + this.ephemeral = ephemeral; + this.setMaxListeners(1000); this.getHook = immutable ? fromJS : (x) => x; - this.env = env; - this.limits = limits; + this.config = config; this.start_seq = start_seq; const table = keys(query)[0]; this.table = table; @@ -95,20 +100,19 @@ export class SyncTableStream extends EventEmitter { const name = patchesStreamName({ string_id: this.string_id }); this.dstream = await dstream({ name, + client: this.client, project_id: this.project_id, - env: this.env, - limits: this.limits, + config: this.config, desc: { path: this.path }, start_seq: this.start_seq, noInventory: this.noInventory, - // ephemeral: true, - // leader: typeof navigator == "undefined", + ephemeral: this.ephemeral, }); this.dstream.on("change", (mesg) => { this.handle(mesg, true); }); this.dstream.on("reject", (err) => { - console.warn("synctable-stream: REJECTED - ", err); + console.warn("synctable-stream: rejected - ", err); }); for (const mesg of this.dstream.getAll()) { this.handle(mesg, false); @@ -141,8 +145,6 @@ export class SyncTableStream extends EventEmitter { const key = this.primaryString(obj); const { string_id, ...obj2 } = obj; if (this.data[key] != null) { - // "You can not change a message once committed to a stream" - // https://github.com/nats-io/nats-server/discussions/4883 throw Error( `object with key ${key} was already written to the stream -- written data cannot be modified`, ); @@ -204,6 +206,8 @@ export class SyncTableStream extends EventEmitter { this.removeAllListeners(); this.dstream?.close(); delete this.dstream; + // @ts-ignore + delete this.client; }; delete = async (_obj) => { diff --git a/src/packages/nats/sync/synctable.ts b/src/packages/conat/sync/synctable.ts similarity index 50% rename from src/packages/nats/sync/synctable.ts rename to src/packages/conat/sync/synctable.ts index bf819c3813..1017e28793 100644 --- a/src/packages/nats/sync/synctable.ts +++ b/src/packages/conat/sync/synctable.ts @@ -1,14 +1,14 @@ -import { type NatsEnv } from "@cocalc/nats/types"; import { SyncTableKV } from "./synctable-kv"; import { SyncTableStream } from "./synctable-stream"; -import { refCacheSync } from "@cocalc/util/refcache"; -import { type KVLimits } from "./general-kv"; -import { type FilteredStreamLimitOptions } from "./stream"; +import refCache from "@cocalc/util/refcache"; +import { type KVLimits } from "./limits"; +import { type FilteredStreamLimitOptions } from "./limits"; import jsonStableStringify from "json-stable-stringify"; +import { type Client } from "@cocalc/conat/core/client"; -export type NatsSyncTable = SyncTableStream | SyncTableKV; +export type ConatSyncTable = SyncTableStream | SyncTableKV; -export type NatsSyncTableFunction = ( +export type ConatSyncTableFunction = ( query: { [table: string]: { [field: string]: any }[] }, options?: { obj?: object; @@ -20,18 +20,18 @@ export type NatsSyncTableFunction = ( // for tables specific to a project, e.g., syncstrings in a project project_id?: string; }, -) => Promise; +) => Promise; -// When the database is watching tables for changefeeds, if it doesn't get a clear expression -// of interest from a client every this much time, it stops managing the changefeed to -// save resources. +// When the database is watching tables for changefeeds, if it doesn't +// get a clear expression of interest from a client every this much time, +// it stops managing the changefeed to save resources. export const CHANGEFEED_INTEREST_PERIOD_MS = 120000; // export const CHANGEFEED_INTEREST_PERIOD_MS = 3000; -interface Options { +export interface SyncTableOptions { query; - env: NatsEnv; + client?: Client; account_id?: string; project_id?: string; atomic?: boolean; @@ -42,18 +42,21 @@ interface Options { desc?: any; start_seq?: number; noInventory?: boolean; + ephemeral?: boolean; } -function createObject(options: Options) { - if (options.stream) { - return new SyncTableStream(options); - } else { - return new SyncTableKV(options); - } -} - -export const createSyncTable = refCacheSync({ +export const createSyncTable = refCache({ name: "synctable", - createKey: (opts) => jsonStableStringify({ ...opts, env: undefined }), - createObject, + createKey: (opts: SyncTableOptions) => + jsonStableStringify({ ...opts, client: undefined })!, + createObject: async (options: SyncTableOptions & { client: Client }) => { + let t; + if (options.stream) { + t = new SyncTableStream(options); + } else { + t = new SyncTableKV(options); + } + await t.init(); + return t; + }, }); diff --git a/src/packages/nats/time.ts b/src/packages/conat/time.ts similarity index 85% rename from src/packages/nats/time.ts rename to src/packages/conat/time.ts index 901bd9c74a..c6b0f297b7 100644 --- a/src/packages/nats/time.ts +++ b/src/packages/conat/time.ts @@ -1,7 +1,7 @@ /* Time sync -- relies on a hub running a time sync server. -IMPORTANT: Our realtime sync algorithm doesn't depend on an accurate clock anymore. +IMPORTANT: Our realtime sync algorithm does NOT depend on an accurate clock anymore. We may use time to compute logical timestamps for convenience, but they will always be increasing and fall back to a non-time sequence for a while in case a clock is out of sync. We do use the time for displaying edit times to users, which is one reason why syncing @@ -17,17 +17,16 @@ In unit testing mode this just falls back to Date.now(). DEVELOPMENT: -See src/packages/backend/nats/test/time.test.ts for relevant unit test, though +See src/packages/backend/conat/test/time.test.ts for relevant unit test, though in test mode this is basically disabled. -Also do this, noting the directory and import of @cocalc/backend/nats. +Also do this, noting the directory and import of @cocalc/backend/conat. ~/cocalc/src/packages/backend$ node Welcome to Node.js v18.17.1. Type ".help" for more information. -> a = require('@cocalc/nats/time'); require('@cocalc/backend/nats') +> a = require('@cocalc/conat/time'); require('@cocalc/backend/conat') { - getEnv: [Getter], getConnection: [Function: debounced], init: [Function: init], getCreds: [AsyncFunction: getCreds] @@ -37,11 +36,10 @@ Type ".help" for more information. */ -import { timeClient } from "@cocalc/nats/service/time"; +import { timeClient } from "@cocalc/conat/service/time"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; -import { getClient } from "@cocalc/nats/client"; +import { getClient } from "@cocalc/conat/client"; import { delay } from "awaiting"; -import { waitUntilConnected } from "./util"; // we use exponential backoff starting with a short interval // then making it longer @@ -79,7 +77,7 @@ async function syncLoop() { } await delay(d); } catch (err) { - console.log(`WARNING: failed to sync clock -- ${err}`); + // console.log(`WARNING: failed to sync clock -- ${err}`); // reset delay d = INTERVAL_START; await delay(d); @@ -97,7 +95,6 @@ export const getSkew = reuseInFlight(async (): Promise => { return skew; } try { - await waitUntilConnected(); const start = Date.now(); const client = getClient(); const tc = timeClient(client); @@ -107,7 +104,7 @@ export const getSkew = reuseInFlight(async (): Promise => { skew = start + rtt / 2 - serverTime; return skew; } catch (err) { - console.log("WARNING: temporary issue syncing time", err); + // console.log("WARNING: temporary issue syncing time", err); skew = 0; return 0; } diff --git a/src/packages/nats/tsconfig.json b/src/packages/conat/tsconfig.json similarity index 100% rename from src/packages/nats/tsconfig.json rename to src/packages/conat/tsconfig.json diff --git a/src/packages/conat/types.ts b/src/packages/conat/types.ts new file mode 100644 index 0000000000..3dc0a311a0 --- /dev/null +++ b/src/packages/conat/types.ts @@ -0,0 +1,11 @@ +export type State = "disconnected" | "connected" | "closed"; + +export interface Location { + project_id?: string; + compute_server_id?: number; + + account_id?: string; + browser_id?: string; + + path?: string; +} diff --git a/src/packages/conat/util.ts b/src/packages/conat/util.ts new file mode 100644 index 0000000000..670fede4d9 --- /dev/null +++ b/src/packages/conat/util.ts @@ -0,0 +1,91 @@ +import jsonStableStringify from "json-stable-stringify"; +import { encode as encodeBase64, decode as decodeBase64 } from "js-base64"; +export { encodeBase64, decodeBase64 }; +import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; + +export function handleErrorMessage(mesg) { + if (mesg?.error) { + if (mesg.error.startsWith("Error: ")) { + throw Error(mesg.error.slice("Error: ".length)); + } else { + throw Error(mesg.error); + } + } + return mesg; +} + +// Returns true if the subject matches the NATS pattern. +export function matchesPattern({ + pattern, + subject, +}: { + pattern: string; + subject: string; +}): boolean { + const subParts = subject.split("."); + const patParts = pattern.split("."); + let i = 0, + j = 0; + while (i < subParts.length && j < patParts.length) { + if (patParts[j] === ">") return true; + if (patParts[j] !== "*" && patParts[j] !== subParts[i]) return false; + i++; + j++; + } + + return i === subParts.length && j === patParts.length; +} + +// Return true if the subject is a valid NATS subject. +// Returns true if the subject is a valid NATS subject (UTF-8 aware) +export function isValidSubject(subject: string): boolean { + if (typeof subject !== "string" || subject.length === 0) return false; + if (subject.startsWith(".") || subject.endsWith(".")) return false; + const tokens = subject.split("."); + // No empty tokens + if (tokens.some((t) => t.length === 0)) return false; + for (let i = 0; i < tokens.length; ++i) { + const tok = tokens[i]; + // ">" is only allowed as last token + if (tok === ">" && i !== tokens.length - 1) return false; + // "*" and ">" are allowed as sole tokens + if (tok !== "*" && tok !== ">") { + // Must not contain "." or any whitespace Unicode code point + if (/[.\s]/u.test(tok)) { + return false; + } + } + // All tokens: must not contain whitespace (unicode aware) + if (/\s/u.test(tok)) { + return false; + } + // Allow any UTF-8 (unicode) chars except dot and whitespace in tokens. + } + return true; +} + +export function isValidSubjectWithoutWildcards(subject: string): boolean { + return ( + isValidSubject(subject) && !subject.includes("*") && !subject.endsWith(">") + ); +} + +export function toKey(x): string | undefined { + if (x === undefined) { + return undefined; + } else if (typeof x === "object") { + return jsonStableStringify(x); + } else { + return `${x}`; + } +} + +// Returns the max payload size for messages for the NATS server +// that we are connected to. This is used for chunking by the kv +// and stream to support arbitrarily large values. +export const getMaxPayload = reuseInFlight(async () => { + // [ ] TODO + return 1e6; +}); + +export const waitUntilConnected = reuseInFlight(async () => {}); diff --git a/src/packages/database/conat/changefeed-api.ts b/src/packages/database/conat/changefeed-api.ts new file mode 100644 index 0000000000..f725236d55 --- /dev/null +++ b/src/packages/database/conat/changefeed-api.ts @@ -0,0 +1,49 @@ +/* + +DEVELOPMENT: + +Turn off conat-server handling for the hub for changefeeds by sending this message from a browser as an admin: + + await cc.client.conat_client.hub.system.terminate({service:'changefeeds'}) + +In a node session: + +DEBUG=cocalc*changefeed* DEBUG_CONSOLE=yes node + + require('@cocalc/backend/conat'); require('@cocalc/database/conat/changefeed-api').init() + +In another session: + + require('@cocalc/backend/conat'); c = require('@cocalc/conat/changefeed/client'); + account_id = '6aae57c6-08f1-4bb5-848b-3ceb53e61ede'; + cf = await c.changefeed({account_id,query:{accounts:[{account_id, first_name:null}]}, heartbeat:5000, lifetime:30000}); + + const {value:{id}} = await cf.next(); + console.log({id}); + for await (const x of cf) { console.log(new Date(), {x}); } + + await c.renew({account_id, id}) +*/ + +import { + changefeedServer, + type ConatSocketServer, +} from "@cocalc/conat/hub/changefeeds"; + +import { db } from "@cocalc/database"; +import { conat } from "@cocalc/backend/conat"; + +let server: ConatSocketServer | null = null; +export function init() { + const D = db(); + server = changefeedServer({ + client: conat(), + userQuery: D.user_query.bind(D), + cancelQuery: (id: string) => D.user_query_cancel_changefeed({ id }), + }); +} + +export function close() { + server?.close(); + server = null; +} diff --git a/src/packages/database/nats/leak-search.ts b/src/packages/database/conat/leak-search.ts similarity index 97% rename from src/packages/database/nats/leak-search.ts rename to src/packages/database/conat/leak-search.ts index a90687e142..842208efc6 100644 --- a/src/packages/database/nats/leak-search.ts +++ b/src/packages/database/conat/leak-search.ts @@ -12,7 +12,7 @@ ACCOUNT_ID="6aae57c6-08f1-4bb5-848b-3ceb53e61ede" DEBUG=cocalc:* DEBUG_CONSOLE=y Then do this - a = require('@cocalc/database/nats/leak-search') + a = require('@cocalc/database/conat/leak-search') await a.testQueryOnly(50) await a.testChangefeed(50) diff --git a/src/packages/database/nats/changefeed-api.ts b/src/packages/database/nats/changefeed-api.ts deleted file mode 100644 index 3fe737a01f..0000000000 --- a/src/packages/database/nats/changefeed-api.ts +++ /dev/null @@ -1,34 +0,0 @@ -/* - -DEVELOPMENT: - -Turn off nats-server handling for the hub for changefeeds by sending this message from a browser as an admin: - - await cc.client.nats_client.hub.system.terminate({service:'changefeeds'}) - -In a node session: - -DEBUG=cocalc*changefeed* DEBUG_CONSOLE=yes node - - require('@cocalc/backend/nats'); require('@cocalc/database/nats/changefeed-api').init() - -In another session: - - require('@cocalc/backend/nats'); c = require('@cocalc/nats/changefeed/client'); - account_id = '6aae57c6-08f1-4bb5-848b-3ceb53e61ede'; - cf = await c.changefeed({account_id,query:{accounts:[{account_id, first_name:null}]}, heartbeat:5000, lifetime:30000}); - - const {value:{id}} = await cf.next(); - console.log({id}); - for await (const x of cf) { console.log(new Date(), {x}); } - - await c.renew({account_id, id}) -*/ - -import { init as initChangefeedServer } from "@cocalc/nats/changefeed/server"; -import { db } from "@cocalc/database"; -import "@cocalc/backend/nats"; - -export function init() { - initChangefeedServer(db); -} diff --git a/src/packages/database/nats/changefeeds.ts b/src/packages/database/nats/changefeeds.ts deleted file mode 100644 index d268a167d3..0000000000 --- a/src/packages/database/nats/changefeeds.ts +++ /dev/null @@ -1,566 +0,0 @@ -/* - -What this does: - -A backend server gets a request that a given changefeed (e.g., "messages" or -"projects" for a given user) needs to be managed. For a while, the server will -watch the datϨabase and put entries in a NATS jetstream kv that represents the -data. The browser also periodically pings the backend saying "I'm still -interested in this changefeed" and the backend server keeps up watching postgres -for changes. When the user is gone for long enough (5 minutes?) the backend -stops watching and just leaves the data as is in NATS. - -When the user comes back, they immediately get the last version of the data -straight from NATS, and their browser says "I'm interested in this changefeed". -The changefeed then gets updated (hopefully 1-2 seconds later) and periodically -updated after that. - - -DEVELOPMENT: - -1. turn off nats-server handling for the hub by sending this message from a browser as an admin: - - await cc.client.nats_client.hub.system.terminate({service:'db'}) - -2. Run this line in nodejs right here: - -DEBUG_CONSOLE=yes DEBUG=cocalc:debug:database:nats:changefeeds - - require("@cocalc/database/nats/changefeeds").init() - - -*/ - -import getLogger from "@cocalc/backend/logger"; -import { JSONCodec } from "nats"; -import userQuery from "@cocalc/database/user-query"; -import { getConnection } from "@cocalc/backend/nats"; -import { getUserId } from "@cocalc/nats/hub-api"; -import { callback } from "awaiting"; -import { db } from "@cocalc/database"; -import { - createSyncTable, - CHANGEFEED_INTEREST_PERIOD_MS as CHANGEFEED_INTEREST_PERIOD_MS_USERS, -} from "@cocalc/nats/sync/synctable"; -import { sha1 } from "@cocalc/backend/misc_node"; -import jsonStableStringify from "json-stable-stringify"; -import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; -import { uuid } from "@cocalc/util/misc"; -import { delay } from "awaiting"; -import { Svcm } from "@nats-io/services"; -import { Coordinator, now } from "./coordinator"; -import { numSubscriptions } from "@cocalc/nats/client"; - -const logger = getLogger("database:nats:changefeeds"); - -const jc = JSONCodec(); - -// How long until the manager's lock on changefeed expires. -// It's good for this to be SHORT, since if a hub-database -// terminates badly (without calling terminate explicitly), then -// nothing else will take over until after this lock expires. -// It's good for this to be LONG, since it reduces load on the system. -// That said, if hubs are killed properly, they release their -// locks on exit. -const LOCK_TIMEOUT_MS = 90000; - -const MAX_MANAGER_CONFLICTS = parseInt( - process.env.COCALC_MAX_MANAGER_CONFLICTS ?? "9999", -); - -// This is a limit on the numChangefeedsBeingCreatedAtOnce: -const PARALLEL_LIMIT = parseInt(process.env.COCALC_PARALLEL_LIMIT ?? "15"); - -const CHANGEFEED_INTEREST_PERIOD_MS = parseInt( - process.env.COCALC_CHANGEFEED_INTEREST_PERIOD_MS ?? `${CHANGEFEED_INTEREST_PERIOD_MS_USERS}`, -); - -export async function init() { - if (process.env.COCALC_TERMINATE_CHANGEFEEDS_ON_EXIT) { - setupExitHandler(); - } - while (true) { - if (terminated) { - return; - } - try { - await mainLoop(); - } catch (err) { - logger.debug(`error running mainLoop -- ${err}`); - } - await delay(15000); - } -} - -let api: any | null = null; -let coordinator: null | Coordinator = null; -async function mainLoop() { - if (terminated) { - return; - } - const subject = "hub.*.*.db"; - logger.debug(`init -- subject='${subject}', options=`); - coordinator = new Coordinator({ timeout: LOCK_TIMEOUT_MS }); - await coordinator.init(); - const nc = await getConnection(); - - // @ts-ignore - const svcm = new Svcm(nc); - - const service = await svcm.add({ - name: "db-server", - version: "0.2.0", - description: "CoCalc Database Service (changefeeds)", - queue: "0", - }); - - api = service.addEndpoint("api", { subject }); - - try { - for await (const mesg of api) { - await handleRequest({ mesg, nc }); - } - } finally { - cancelAllChangefeeds(); - try { - await coordinator?.close(); - } catch (err) { - logger.debug("error closing coordinator", err); - } - coordinator = null; - } -} - -// try very hard to call terminate properly, so can locks are freed -// and clients don't have to wait for the locks to expire. -function setupExitHandler() { - async function exitHandler(evtOrExitCodeOrError: number | string | Error) { - try { - await terminate(); - } catch (e) { - console.error("EXIT HANDLER ERROR", e); - } - - process.exit(isNaN(+evtOrExitCodeOrError) ? 1 : +evtOrExitCodeOrError); - } - [ - "beforeExit", - "uncaughtException", - "unhandledRejection", - "SIGHUP", - "SIGINT", - "SIGQUIT", - "SIGILL", - "SIGTRAP", - "SIGABRT", - "SIGBUS", - "SIGFPE", - "SIGUSR1", - "SIGSEGV", - "SIGUSR2", - "SIGTERM", - "exit", - ].forEach((evt) => process.on(evt, exitHandler)); -} - -let terminated = false; -export async function terminate() { - if (terminated) { - return; - } - console.log("changefeeds: TERMINATE"); - logger.debug("terminating service"); - terminated = true; - api?.stop(); - api = null; - cancelAllChangefeeds(); - if (coordinator != null) { - console.log("about to try to async save"); - await coordinator.save(); - console.log(coordinator?.dkv?.hasUnsavedChanges()); - await coordinator?.close(); - console.log("coordinator successfully saved"); - coordinator = null; - } -} - -let numRequestsAtOnce = 0; -let numChangefeedsBeingCreatedAtOnce = 0; -async function handleRequest({ mesg, nc }) { - let resp; - try { - numRequestsAtOnce += 1; - logger.debug("handleRequest", { - numRequestsAtOnce, - numSubscriptions: numSubscriptions(), - numChangefeedsBeingCreatedAtOnce, - numChangefeedsBeingManaging: Object.keys(changefeedHashes).length, - numCanceledSinceStart, - }); - const { account_id, project_id } = getUserId(mesg.subject); - const { name, args } = jc.decode(mesg.data) ?? ({} as any); - //console.log(`got request: "${JSON.stringify({ name, args })}"`); - // logger.debug(`got request: "${JSON.stringify({ name, args })}"`); - if (!name) { - throw Error("api endpoint name must be given in message"); - } - // logger.debug("handling server='db' request:", { - // account_id, - // project_id, - // name, - // }); - resp = await getResponse({ - name, - args, - account_id, - project_id, - nc, - }); - } catch (err) { - logger.debug(`ERROR -- ${err}`); - resp = { error: `${err}` }; - } finally { - numRequestsAtOnce -= 1; - } - // logger.debug(`Responding with "${JSON.stringify(resp)}"`); - mesg.respond(jc.encode(resp)); -} - -async function getResponse({ name, args, account_id, project_id, nc }) { - if (name == "userQuery") { - const opts = { ...args[0], account_id, project_id }; - if (!opts.changes) { - // a normal query - console.log("doing normal userQuery", opts); - return await userQuery(opts); - } else { - return await createChangefeed(opts, nc); - } - } else { - throw Error(`name='${name}' not implemented`); - } -} - -function queryTable(query) { - return Object.keys(query)[0]; -} - -// changefeedHashes maps changes (database changefeed id) to hash -const changefeedHashes: { [id: string]: string } = {}; -// changefeedChanges maps hash to changes. -const changefeedChanges: { [hash: string]: string } = {}; -// changefeedInterest maps hash to time -const changefeedInterest: { [hash: string]: number } = {}; -// changefeedSynctables maps hash to SyncTable -const changefeedSynctables: { [hash: string]: any } = {}; -const changefeedManagerConflicts: { [id: string]: number } = {}; - -let numCanceledSinceStart = 0; -async function cancelChangefeed({ - hash, - changes, -}: { - hash?: string; - changes?: string; -}) { - logger.debug("cancelChangefeed", { changes, hash }); - numCanceledSinceStart += 1; - if (changes && !hash) { - hash = changefeedHashes[changes]; - } else if (hash && !changes) { - changes = changefeedChanges[hash]; - } else { - // nothing - return; - } - if (!hash || !changes) { - // already canceled - return; - } - coordinator?.unlock(hash); - const synctable = changefeedSynctables[hash]; - delete changefeedSynctables[hash]; - delete changefeedInterest[hash]; - delete changefeedHashes[changes]; - delete changefeedChanges[hash]; - delete changefeedManagerConflicts[hash]; - db().user_query_cancel_changefeed({ id: changes }); - if (synctable) { - try { - await synctable.close(); - } catch (err) { - logger.debug(`WARNING: error closing changefeed synctable -- ${err}`, { - hash, - }); - } - } -} - -function cancelAllChangefeeds() { - logger.debug("cancelAllChangefeeds"); - for (const changes in changefeedHashes) { - cancelChangefeed({ changes }); - } -} - -// This is tricky. We return the first result as a normal -// async function, but then handle (and don't return) -// the subsequent calls to cb generated by the changefeed. -const createChangefeed = reuseInFlight( - async (opts, nc) => { - const query = opts.query; - // the query *AND* the user making it define the thing: - const user = { account_id: opts.account_id, project_id: opts.project_id }; - const desc = jsonStableStringify({ - query, - ...user, - })!; - const hash = sha1(desc); - if (coordinator == null) { - logger.debug("coordinator is not defined"); - return; - } - - // ALWAYS update that a user is interested in this changefeed - coordinator.updateUserInterest(hash); - - const manager = coordinator.getManagerId(hash); - logger.debug("createChangefeed -- considering: ", { - table: queryTable(query), - hash, - managerId: coordinator.managerId, - manager, - }); - if (manager && coordinator.managerId != manager) { - logger.debug(`somebody else ${manager} is the manager`, { hash }); - if (changefeedInterest[hash]) { - changefeedManagerConflicts[hash] = - (changefeedManagerConflicts[hash] ?? 0) + 1; - logger.debug( - `both us (${coordinator.managerId}) and ${manager} we are also managing changefeed`, - { - hash, - count: changefeedManagerConflicts[hash], - max: MAX_MANAGER_CONFLICTS, - }, - ); - if (changefeedManagerConflicts[hash] >= MAX_MANAGER_CONFLICTS) { - cancelChangefeed({ hash }); - } - return; - } - return; - } - // take it - coordinator.lock(hash); - - if (changefeedInterest[hash]) { - changefeedInterest[hash] = now(); - logger.debug("use existing changefeed", { - hash, - table: queryTable(query), - user, - }); - } else { - // we create new changefeeed but do NOT block on this. While creating this - // if user calls again then changefeedInterest[hash] is set, so it'll just - // use the existing changefeed (the case above). If things eventually go awry, - // changefeedInterest[hash] gets cleared. - createNewChangefeed({ query, user, nc, opts, hash }); - } - }, - { createKey: (args) => jsonStableStringify(args[0])! }, -); - -const createNewChangefeed = async ({ query, user, nc, opts, hash }) => { - logger.debug("create new changefeed", queryTable(query), user); - changefeedInterest[hash] = now(); - const changes = uuid(); - changefeedHashes[changes] = hash; - changefeedChanges[hash] = changes; - logger.debug( - "managing ", - Object.keys(changefeedHashes).length, - "changefeeds", - ); - const env = { nc, jc, sha1 }; - - let done = false; - // we start watching state immediately and updating it, since if it - // takes a while to setup the feed, we don't want somebody else to - // steal it. - const watchManagerState = async () => { - while (!done && changefeedInterest[hash]) { - await delay(LOCK_TIMEOUT_MS / 1.5); - if (done) { - return; - } - if (coordinator == null) { - done = true; - return; - } - const manager = coordinator.getManagerId(hash); - if (manager != coordinator.managerId) { - // we are no longer the manager - cancelChangefeed({ changes }); - done = true; - return; - } - // update the lock - coordinator.lock(hash); - } - }; - watchManagerState(); - - // If you change any settings below (i.e., atomic or immutable), you might also have to change them in - // src/packages/sync/table/changefeed-nats.ts - const synctable = createSyncTable({ - query, - env, - account_id: opts.account_id, - project_id: opts.project_id, - // atomic = false is just way too slow due to the huge number of distinct - // messages, which NATS is not as good with. - atomic: true, - immutable: false, - }); - changefeedSynctables[hash] = synctable; - - // before doing the HARD WORK, we wait until there aren't too many - // other "threads" doing hard work: - while (numChangefeedsBeingCreatedAtOnce >= PARALLEL_LIMIT) { - // TODO: This is STUPID - await delay(25); - } - try { - numChangefeedsBeingCreatedAtOnce += 1; - try { - await synctable.init(); - logger.debug("successfully created synctable", queryTable(query), user); - } catch (err) { - logger.debug(`Error initializing changefeed -- ${err}`, { hash }); - cancelChangefeed({ changes }); - } - - const handleFirst = ({ cb, err, rows }) => { - if (err || rows == null) { - cb(err ?? "missing result"); - return; - } - try { - if (synctable.get_state() != "connected") { - cb("not connected"); - return; - } - const current = synctable.get(); - const databaseKeys = new Set(); - for (const obj of rows) { - databaseKeys.add(synctable.getKey(obj)); - synctable.set(obj); - } - for (const key in current) { - if (!databaseKeys.has(key)) { - // console.log("remove from synctable", key); - synctable.delete(key); - } - } - cb(); - } catch (err) { - logger.debug(`Error handling first changefeed output -- ${err}`, { - hash, - }); - cb(err); - } - }; - - const handleUpdate = ({ action, new_val, old_val }) => { - // action = 'insert', 'update', 'delete', 'close' - // e.g., {"action":"insert","new_val":{"title":"testingxxxxx","project_id":"81e0c408-ac65-4114-bad5-5f4b6539bd0e"}} - const obj = new_val ?? old_val; - if (obj == null) { - // nothing we can do with this - return; - } - if (action == "insert" || action == "update") { - const cur = synctable.get(new_val); - // logger.debug({ table: queryTable(query), action, new_val, old_val }); - synctable.set({ ...cur, ...new_val }); - } else if (action == "delete") { - synctable.delete(old_val); - } else if (action == "close") { - cancelChangefeed({ changes }); - } - }; - - const f = (cb) => { - let first = true; - db().user_query({ - ...opts, - changes, - cb: (err, x) => { - if (first) { - first = false; - handleFirst({ cb, err, rows: x?.[synctable.table] }); - return; - } - try { - handleUpdate(x as any); - } catch (err) { - logger.debug(`Error handling update: ${err}`, { hash }); - cancelChangefeed({ changes }); - } - }, - }); - }; - try { - await callback(f); - // it's running successfully - changefeedInterest[hash] = now(); - - const watchUserInterest = async () => { - logger.debug("watchUserInterest", { hash }); - // it's all setup and running. If there's no interest for a while, stop watching - while (!done && changefeedInterest[hash]) { - await delay(CHANGEFEED_INTEREST_PERIOD_MS); - if (done) { - break; - } - if ( - now() - changefeedInterest[hash] > - CHANGEFEED_INTEREST_PERIOD_MS - ) { - logger.debug("watchUserInterest: no local interest", { - hash, - }); - // we check both the local known interest *AND* interest recorded - // by any other servers! - const last = coordinator?.getUserInterest(hash) ?? 0; - if (now() - last >= CHANGEFEED_INTEREST_PERIOD_MS) { - logger.debug("watchUserInterest: no interest, canceling", { - hash, - }); - cancelChangefeed({ changes }); - done = true; - break; - } - } - } - logger.debug("watchUserInterest: stopped watching since done", { - hash, - }); - }; - // do not block on this. - watchUserInterest(); - } catch (err) { - // if anything goes wrong, make sure we don't think the changefeed is working. - cancelChangefeed({ changes }); - logger.debug( - `WARNING: error creating changefeed -- ${err}`, - queryTable(query), - user, - ); - } - } finally { - numChangefeedsBeingCreatedAtOnce -= 1; - } -}; diff --git a/src/packages/database/nats/coordinator.ts b/src/packages/database/nats/coordinator.ts deleted file mode 100644 index 4f5e2864be..0000000000 --- a/src/packages/database/nats/coordinator.ts +++ /dev/null @@ -1,167 +0,0 @@ -/* -This is for managing who is responsible for each changefeed. - -It stores: - -- for each changefeed id, the managerId of who is manging it -- for each manager id, time when it last checked in - -The manger checks in 2.5x every timeout period. -If a manger doesn't check in for the entire timeout period, then -they are considered gone. - -DEVELOPMENT: - -c = await require('@cocalc/database/nats/coordinator').coordinator({timeout:10000}) - -*/ - -import { dkv, type DKV } from "@cocalc/backend/nats/sync"; -import { randomId } from "@cocalc/nats/names"; -import getTime from "@cocalc/nats/time"; - -interface Entry { - // last time user expressed interest in this changefeed - user?: number; - // manager of this changefeed. - managerId?: string; - // last time manager updated lock on this changefeed - lock?: number; -} - -function mergeTime( - a: number | undefined, - b: number | undefined, -): number | undefined { - // time of interest should clearly always be the largest known value so far. - if (a == null && b == null) { - return undefined; - } - return Math.max(a ?? 0, b ?? 0); -} - -// TODO: note -- local or remote may be null -- fix this! -function resolveMergeConflict(local?: Entry, remote?: Entry): Entry { - const user = mergeTime(remote?.user, local?.user); - let managerId = local?.managerId ?? remote?.managerId; - if ( - local?.managerId && - remote?.managerId && - local.managerId != remote.managerId - ) { - // conflicting manager - winner is one with newest lock. - if ((local.lock ?? 0) > (remote.lock ?? 0)) { - managerId = local.managerId; - } else { - managerId = remote.managerId; - } - } - const lock = mergeTime(remote?.lock, local?.lock); - return { user, lock, managerId }; -} - -export const now = () => getTime({ noError: true }); - -const LIMITS = { - // discard any keys that are 15 minutes old -- the lock and user interest - // updates are much more frequently than this, but this keeps memory usage down. - max_age: 1000 * 60 * 15, -}; - -export async function coordinator(opts) { - const C = new Coordinator(opts); - await C.init(); - return C; -} - -export class Coordinator { - public readonly managerId: string; - public dkv?: DKV; - - // if a manager hasn't update that it is managing this changefeed for timeout ms, then - // the lock is relinquished. - public readonly timeout: number; - - constructor({ timeout }: { timeout: number }) { - this.managerId = randomId(); - this.timeout = timeout; - } - - init = async () => { - this.dkv = await dkv({ - name: "changefeed-manager", - limits: LIMITS, - merge: ({ local, remote }) => resolveMergeConflict(local, remote), - }); - }; - - save = async () => { - await this.dkv?.save(); - }; - - close = async () => { - await this.dkv?.close(); - delete this.dkv; - }; - - getManagerId = (id: string): string | undefined => { - if (this.dkv == null) { - throw Error("coordinator is closed"); - } - const cur = this.dkv.get(id); - if (cur == null) { - return; - } - const { managerId, lock } = cur; - if (!managerId || !lock) { - return undefined; - } - if (lock < now() - this.timeout) { - // lock is too old - return undefined; - } - return managerId; - }; - - // update that this manager has the lock on this changefeed. - lock = (id: string) => { - if (this.dkv == null) { - throw Error("coordinator is closed"); - } - this.dkv.set(id, { - ...this.dkv.get(id), - lock: now(), - managerId: this.managerId, - }); - }; - - // ensure that this manager no longer has the lock - unlock = (id: string) => { - if (this.dkv == null) { - throw Error("coordinator is closed"); - } - const x: Entry = this.dkv.get(id) ?? {}; - if (x.managerId == this.managerId) { - // we are the manager - this.dkv.set(id, { ...x, lock: 0, managerId: "" }); - return; - } - }; - - // user expresses interest in changefeed with given id, - // which we may or may not be the manager of. - updateUserInterest = (id: string) => { - if (this.dkv == null) { - throw Error("coordinator is closed"); - } - this.dkv.set(id, { ...this.dkv.get(id), user: now() }); - }; - - getUserInterest = (id: string): number => { - if (this.dkv == null) { - throw Error("coordinator is closed"); - } - const { user } = this.dkv.get(id) ?? {}; - return user ?? 0; - }; -} diff --git a/src/packages/database/package.json b/src/packages/database/package.json index c5c02e9e5f..15b29d94fe 100644 --- a/src/packages/database/package.json +++ b/src/packages/database/package.json @@ -5,7 +5,7 @@ "exports": { ".": "./dist/index.js", "./accounts/*": "./dist/accounts/*.js", - "./nats/*": "./dist/nats/*.js", + "./conat/*": "./dist/conat/*.js", "./pool": "./dist/pool/index.js", "./pool/*": "./dist/pool/*.js", "./postgres/*": "./dist/postgres/*.js", @@ -19,31 +19,25 @@ }, "dependencies": { "@cocalc/backend": "workspace:*", + "@cocalc/conat": "workspace:*", "@cocalc/database": "workspace:*", - "@cocalc/nats": "workspace:*", "@cocalc/util": "workspace:*", - "@nats-io/services": "3.0.0", "async": "^1.5.2", "awaiting": "^3.0.0", "debug": "^4.4.0", "immutable": "^4.3.0", - "json-stable-stringify": "^1.0.1", "lodash": "^4.17.21", "lru-cache": "^7.18.3", - "nats": "^2.29.3", - "node-fetch": "2.6.7", "pg": "^8.7.1", "random-key": "^0.3.2", "read": "^1.0.7", "sql-string-escape": "^1.1.6", - "uuid": "^8.3.2", "validator": "^13.6.0" }, "devDependencies": { - "@types/node": "^18.16.14", "@types/lodash": "^4.14.202", + "@types/node": "^18.16.14", "@types/pg": "^8.6.1", - "@types/uuid": "^8.3.1", "coffeescript": "^2.5.1" }, "scripts": { @@ -52,6 +46,7 @@ "clean": "rm -rf dist", "tsc": "../node_modules/.bin/tsc --watch --pretty --preserveWatchOutput", "test": "pnpm exec jest --forceExit --runInBand", + "depcheck": "pnpx depcheck | grep -Ev '\\.coffee|coffee$'", "prepublishOnly": "pnpm test" }, "repository": { @@ -59,7 +54,10 @@ "url": "https://github.com/sagemathinc/cocalc" }, "homepage": "https://github.com/sagemathinc/cocalc", - "keywords": ["postgresql", "cocalc"], + "keywords": [ + "postgresql", + "cocalc" + ], "author": "SageMath, Inc.", "license": "SEE LICENSE.md", "bugs": { diff --git a/src/packages/database/postgres-server-queries.coffee b/src/packages/database/postgres-server-queries.coffee index e995ae09e7..423fedbc84 100644 --- a/src/packages/database/postgres-server-queries.coffee +++ b/src/packages/database/postgres-server-queries.coffee @@ -40,7 +40,6 @@ read = require('read') {PROJECT_COLUMNS, one_result, all_results, count_result, expire_time} = require('./postgres-base') -{syncdoc_history} = require('./postgres/syncdoc-history') # TODO is set_account_info_if_possible used here?! {is_paying_customer, set_account_info_if_possible} = require('./postgres/account-queries') {getStripeCustomerId, syncCustomer} = require('./postgres/stripe') @@ -2402,19 +2401,6 @@ exports.extend_PostgreSQL = (ext) -> class PostgreSQL extends ext cb : cb ], opts.cb) - syncdoc_history: (opts) => - opts = defaults opts, - string_id : required - patches : false # if true, include actual patches - cb : required - try - opts.cb(undefined, await syncdoc_history(@, opts.string_id, opts.patches)) - catch err - opts.cb(err) - - syncdoc_history_async : (string_id, patches) => - return await syncdoc_history(@, string_id, patches) - # async function site_license_usage_stats: () => return await site_license_usage_stats(@) diff --git a/src/packages/database/postgres-user-queries.coffee b/src/packages/database/postgres-user-queries.coffee index 91af12a44d..2cb8f7754d 100644 --- a/src/packages/database/postgres-user-queries.coffee +++ b/src/packages/database/postgres-user-queries.coffee @@ -379,7 +379,7 @@ exports.extend_PostgreSQL = (ext) -> class PostgreSQL extends ext else if opts.account_id? dbg = r.dbg = @_dbg("user_set_query(account_id='#{opts.account_id}', table='#{opts.table}')") else - return {err:"FATAL: account_id or project_id must be specified"} + return {err:"FATAL: account_id or project_id must be specified to set query on table='#{opts.table}'"} if not SCHEMA[opts.table]? return {err:"FATAL: table '#{opts.table}' does not exist"} @@ -1168,7 +1168,7 @@ exports.extend_PostgreSQL = (ext) -> class PostgreSQL extends ext subs[value] = user_query.project_id cb() else - cb("FATAL: you do not have read access to this project") + cb("FATAL: you do not have read access to this project -- account_id=#{account_id}, project_id_=#{project_id}") when 'project_id-public' if not user_query.project_id? cb("FATAL: must specify project_id") diff --git a/src/packages/database/postgres/syncdoc-history.ts b/src/packages/database/postgres/syncdoc-history.ts deleted file mode 100644 index a030c8f715..0000000000 --- a/src/packages/database/postgres/syncdoc-history.ts +++ /dev/null @@ -1,106 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -import { callback2 } from "@cocalc/util/async-utils"; -import { trunc } from "@cocalc/util/misc"; -import { PostgreSQL } from "./types"; - -export interface Patch { - time_utc: Date; - patch_length?: number; - patch?: string; - user?: string; - account_id?: string; - format?: number; - snapshot?: string; -} - -type User = { account_id: string; user: string }; - -async function get_users(db: PostgreSQL, where): Promise { - const query = "SELECT project_id, users FROM syncstrings"; - // get the user_id --> account_id map - const results = await callback2(db._query, { query, where }); - if (results.rows.length != 1) { - throw Error("no such syncstring"); - } - const account_ids: string[] = results.rows[0].users - ? results.rows[0].users - : []; // syncdoc exists, but not used yet. - const project_id: string = results.rows[0].project_id; - const project_title: string = trunc( - ( - await callback2(db.get_project, { - columns: ["title"], - project_id, - }) - ).title, - 80, - ); - - // get the names of the users - // TODO: this whole file should be in @cocalc/server, be an api endpoint for api/v2, - // and this code below should instead use @cocalc/server/accounts/get-name:getNames. - const names = await callback2(db.account_ids_to_usernames, { account_ids }); - const users: User[] = []; - for (const account_id of account_ids) { - if (account_id == project_id) { - users.push({ account_id, user: `Project: ${project_title}` }); - continue; - } - const name = names[account_id]; - if (name == null) continue; - const user = trunc(`${name.first_name} ${name.last_name}`, 80); - users.push({ account_id, user }); - } - return users; -} - -export async function syncdoc_history( - db: PostgreSQL, - string_id: string, - include_patches: boolean = false, -): Promise { - const where = { "string_id = $::CHAR(40)": string_id }; - const users: User[] = await get_users(db, where); - - const order_by = "time"; - let query: string; - if (include_patches) { - query = "SELECT time, user_id, format, patch, snapshot FROM patches"; - } else { - query = - "SELECT time, user_id, format, length(patch) as patch_length FROM patches"; - } - const results = await callback2(db._query, { - query, - where, - order_by, - timeout_s: 300, - }); - const patches: Patch[] = []; - function format_patch(row): Patch { - const patch: Patch = { time_utc: row.time, format: row.format }; - const u = users[row.user_id]; - if (u != null) { - for (const k in u) { - patch[k] = u[k]; - } - } - if (include_patches) { - patch.patch = row.patch; - if (row.snapshot != null) { - patch.snapshot = row.snapshot; - } - } else { - patch.patch_length = row.patch_length; - } - return patch; - } - for (const row of results.rows) { - patches.push(format_patch(row)); - } - return patches; -} diff --git a/src/packages/database/postgres/test-postgres-import.js b/src/packages/database/postgres/test-postgres-import.js deleted file mode 100644 index 3d307b1f8a..0000000000 --- a/src/packages/database/postgres/test-postgres-import.js +++ /dev/null @@ -1,10 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -require("coffeescript/register"); -require("ts-node").register({ - cacheDirectory: process.env.HOME + "/.ts-node-cache", -}); -require("../postgres.coffee"); diff --git a/src/packages/database/postgres/types.ts b/src/packages/database/postgres/types.ts index 9b290f3798..b952ad755e 100644 --- a/src/packages/database/postgres/types.ts +++ b/src/packages/database/postgres/types.ts @@ -76,7 +76,6 @@ export interface DeletePassportOpts { account_id: string; strategy: string; // our name of the strategy id: string; - cb?: CB; } export interface PassportExistsOpts { @@ -240,7 +239,7 @@ export interface PostgreSQL extends EventEmitter { create_passport(opts: CreatePassportOpts): Promise; - delete_passport(opts: DeletePassportOpts): void; + delete_passport(opts: DeletePassportOpts): Promise; set_passport_settings( db: PostgreSQL, @@ -333,8 +332,6 @@ export interface PostgreSQL extends EventEmitter { cb: CB; }): void; - syncdoc_history_async(string_id: string, patches?: boolean): void; - set_project_state(opts: { project_id: string; state: ProjectState; @@ -382,6 +379,33 @@ export interface PostgreSQL extends EventEmitter { cutoff?: Date; cb?: CB; }); + + when_sent_project_invite(opts: { project_id: string; to: string; cb?: CB }); + + sent_project_invite(opts: { + project_id: string; + to: string; + error?: string; + cb?: CB; + }); + + account_creation_actions(opts: { + email_address: string; + action?: any; + ttl?: number; + cb: CB; + }); + + log_client_error(opts: { + event: string; + error: string; + account_id?: string; + cb?: CB; + }); + + webapp_error(opts: object); + + set_project_settings(opts: { project_id: string; settings: object; cb?: CB }); } // This is an extension of BaseProject in projects/control/base.ts diff --git a/src/packages/database/settings/customize.ts b/src/packages/database/settings/customize.ts index a78cf06dc3..11170fbf0d 100644 --- a/src/packages/database/settings/customize.ts +++ b/src/packages/database/settings/customize.ts @@ -127,6 +127,16 @@ export default async function getCustomize( strategies, verifyEmailAddresses: settings.verify_emails && settings.email_enabled, + + version: { + min_project: parseInt(settings.version_min_project), + min_compute_server: parseInt(settings.version_min_compute_server), + min_browser: parseInt(settings.version_min_browser), + recommended_browser: parseInt(settings.version_recommended_browser), + compute_server_min_project: parseInt( + settings.version_compute_server_min_project, + ), + }, }; } return fields ? copy_with(cachedCustomize, fields) : cachedCustomize; diff --git a/src/packages/database/tsconfig.json b/src/packages/database/tsconfig.json index a9234ba729..0121dd9584 100644 --- a/src/packages/database/tsconfig.json +++ b/src/packages/database/tsconfig.json @@ -8,7 +8,7 @@ "exclude": ["node_modules", "dist"], "references": [ { "path": "../backend" }, - { "path": "../nats" }, + { "path": "../conat" }, { "path": "../util" } ] } diff --git a/src/packages/file-server/package.json b/src/packages/file-server/package.json index 0665a655dc..835280d7dc 100644 --- a/src/packages/file-server/package.json +++ b/src/packages/file-server/package.json @@ -10,23 +10,24 @@ "preinstall": "npx only-allow pnpm", "build": "pnpm exec tsc --build", "tsc": "pnpm exec tsc --watch --pretty --preserveWatchOutput", - "test": "pnpm exec jest --runInBand" + "test": "pnpm exec jest --runInBand", + "depcheck": "pnpx depcheck" }, "files": ["dist/**", "README.md", "package.json"], "author": "SageMath, Inc.", - "keywords": ["utilities", "nats", "cocalc"], + "keywords": ["utilities", "btrfs", "zfs", "cocalc"], "license": "SEE LICENSE.md", "dependencies": { "@cocalc/backend": "workspace:*", + "@cocalc/conat": "workspace:*", "@cocalc/file-server": "workspace:*", - "@cocalc/nats": "workspace:*", "@cocalc/util": "workspace:*", "awaiting": "^3.0.0", - "better-sqlite3": "^11.8.1", + "better-sqlite3": "^11.10.0", "lodash": "^4.17.21" }, "devDependencies": { - "@types/better-sqlite3": "^7.6.12", + "@types/better-sqlite3": "^7.6.13", "@types/lodash": "^4.14.202" }, "repository": { diff --git a/src/packages/file-server/tsconfig.json b/src/packages/file-server/tsconfig.json index 9d5e2b1337..c2fcccc371 100644 --- a/src/packages/file-server/tsconfig.json +++ b/src/packages/file-server/tsconfig.json @@ -5,5 +5,5 @@ "outDir": "dist" }, "exclude": ["node_modules", "dist", "test"], - "references": [{ "path": "../util", "path": "../nats", "path": "../backend" }] + "references": [{ "path": "../util", "path": "../conat", "path": "../backend" }] } diff --git a/src/packages/file-server/zfs/names.ts b/src/packages/file-server/zfs/names.ts index aee7b41cd5..a9f5ff7571 100644 --- a/src/packages/file-server/zfs/names.ts +++ b/src/packages/file-server/zfs/names.ts @@ -1,7 +1,7 @@ import { join } from "path"; import { context } from "./config"; import { primaryKey, type PrimaryKey } from "./types"; -import { randomId } from "@cocalc/nats/names"; +import { randomId } from "@cocalc/conat/names"; export function databaseFilename(data: string) { return join(data, "database.sqlite3"); diff --git a/src/packages/frontend/account/account-page.tsx b/src/packages/frontend/account/account-page.tsx index 6d649d944c..acea76d229 100644 --- a/src/packages/frontend/account/account-page.tsx +++ b/src/packages/frontend/account/account-page.tsx @@ -332,17 +332,19 @@ export const AccountPage: React.FC = () => { ); }; +declare var DEBUG; + function RedirectToNextApp({}) { const isMountedRef = useIsMountedRef(); useEffect(() => { const f = () => { - if (isMountedRef.current) { + if (isMountedRef.current && !DEBUG) { // didn't get signed in so go to landing page window.location.href = appBasePath; } }; - setTimeout(f, 5000); + setTimeout(f, 10000); }, []); return ; diff --git a/src/packages/frontend/account/actions.ts b/src/packages/frontend/account/actions.ts index d9745bbb22..1cffbe06b2 100644 --- a/src/packages/frontend/account/actions.ts +++ b/src/packages/frontend/account/actions.ts @@ -3,14 +3,12 @@ * License: MS-RSL – see LICENSE.md for details */ -import { fromJS } from "immutable"; import { join } from "path"; import { alert_message } from "@cocalc/frontend/alerts"; import { AccountClient } from "@cocalc/frontend/client/account"; import api from "@cocalc/frontend/client/api"; import { appBasePath } from "@cocalc/frontend/customize/app-base-path"; import { set_url } from "@cocalc/frontend/history"; -import { track_conversion } from "@cocalc/frontend/misc"; import { deleteRememberMe } from "@cocalc/frontend/misc/remember-me"; import track from "@cocalc/frontend/user-tracking"; import { webapp_client } from "@cocalc/frontend/webapp-client"; @@ -33,10 +31,6 @@ export class AccountActions extends Actions { this.processSignUpTags(); } - private help(): string { - return this.redux.getStore("customize").get("help_email"); - } - derive_show_global_info(store: AccountStore): void { // TODO when there is more time, rewrite this to be tied to announcements of a specific type (and use their timestamps) // for now, we use the existence of a timestamp value to indicate that the banner is not shown @@ -82,111 +76,8 @@ export class AccountActions extends Actions { }); } - public async sign_in(email: string, password: string): Promise { - const doc_conn = - "[connectivity debugging tips](https://doc.cocalc.com/howto/connectivity-issues.html)"; - const err_help = `\ -Please try again. - -If that doesn't work after a few minutes, try these ${doc_conn} or email ${this.help()}.\ -`; - - this.setState({ signing_in: true }); - let mesg; - try { - mesg = await this.account_client.sign_in({ - email_address: email, - password, - remember_me: true, - get_api_key: !!this.redux.getStore("page").get("get_api_key"), - }); - } catch (err) { - this.setState({ - sign_in_error: `There was an error signing you in -- (${err.message}). ${err_help}`, - }); - return; - } - this.setState({ signing_in: false }); - switch (mesg.event) { - case "sign_in_failed": - this.setState({ sign_in_error: mesg.reason }); - return; - case "signed_in": - break; - case "error": - this.setState({ sign_in_error: mesg.reason }); - return; - default: - // should never ever happen - this.setState({ - sign_in_error: `The server responded with invalid message when signing in: ${JSON.stringify( - mesg, - )}`, - }); - return; - } - } - - public async create_account( - first_name: string, - last_name: string, - email_address: string, - password: string, - token?: string, - usage_intent?: string, - ): Promise { - this.setState({ signing_up: true }); - let mesg; - try { - mesg = await this.account_client.create_account({ - first_name, - last_name, - email_address, - password, - usage_intent, - agreed_to_terms: true, // since never gets called if not set in UI - token, - get_api_key: !!this.redux.getStore("page").get("get_api_key"), - }); - } catch (err) { - // generic error. - this.setState( - fromJS({ sign_up_error: { generic: JSON.stringify(err) } }) as any, - ); - return; - } finally { - this.setState({ signing_up: false }); - } - switch (mesg.event) { - case "account_creation_failed": - this.setState({ sign_up_error: mesg.reason }); - return; - case "signed_in": - this.redux.getActions("page").set_active_tab("projects"); - track_conversion("create_account"); - return; - default: - // should never ever happen - alert_message({ - type: "error", - message: `The server responded with invalid message to account creation request: #{JSON.stringify(mesg)}`, - }); - } - } - // deletes the account and then signs out everywhere public async delete_account(): Promise { - // cancel any subscriptions - try { - await this.redux.getActions("billing").cancel_everything(); - } catch (err) { - if (this.redux.getStore("billing").get("no_stripe")) { - // stripe not configured on backend, so this err is expected - } else { - throw err; - } - } - try { // actually request to delete the account // this should return {status: "success"} @@ -200,40 +91,6 @@ If that doesn't work after a few minutes, try these ${doc_conn} or email ${this. this.sign_out(true); } - public async forgot_password(email_address: string): Promise { - try { - await this.account_client.forgot_password(email_address); - } catch (err) { - this.setState({ - forgot_password_error: `Error sending password reset message to ${email_address} -- ${err}. Write to ${this.help()} for help.`, - forgot_password_success: "", - }); - return; - } - this.setState({ - forgot_password_success: `Password reset message sent to ${email_address}; if you don't receive it, check your spam folder; if you have further trouble, write to ${this.help()}.`, - forgot_password_error: "", - }); - } - - public async reset_password( - reset_code: string, - new_password: string, - ): Promise { - try { - await this.account_client.reset_forgot_password(reset_code, new_password); - } catch (err) { - this.setState({ - reset_password_error: err.message, - }); - return; - } - // success - // TODO: can we automatically log them in? Should we? Seems dangerous. - history.pushState({}, "", location.href); - this.setState({ reset_key: "", reset_password_error: "" }); - } - public async sign_out( everywhere: boolean, sign_in: boolean = false, diff --git a/src/packages/frontend/account/init.ts b/src/packages/frontend/account/init.ts index 8a629677d9..426dd972a6 100644 --- a/src/packages/frontend/account/init.ts +++ b/src/packages/frontend/account/init.ts @@ -13,6 +13,7 @@ import { init_dark_mode } from "./dark-mode"; import { reset_password_key } from "../client/password-reset"; import { hasRememberMe } from "@cocalc/frontend/misc/remember-me"; import { appBasePath } from "@cocalc/frontend/customize/app-base-path"; +import { once } from "@cocalc/util/async-utils"; export function init(redux) { // Register account store @@ -43,22 +44,29 @@ export function init(redux) { actions.setState({ reset_key: reset_password_key() }); // Login status - webapp_client.on("signed_in", function (mesg) { + webapp_client.on("signed_in", async (mesg) => { if (mesg?.api_key) { // wait for sign in to finish and cookie to get set, then redirect setTimeout(() => { window.location.href = `https://authenticated?api_key=${mesg.api_key}`; }, 2000); } + const table = redux.getTable("account")._table; + if (table != "connected") { + // not fully signed in until the account table is connected, so that we know + // email address, etc. If we don't set this, the UI thinks the user is anonymous + // for a second, which is disconcerting. + await once(table, "connected"); + } redux.getActions("account").set_user_type("signed_in"); }); webapp_client.on("signed_out", () => - redux.getActions("account").set_user_type("public") + redux.getActions("account").set_user_type("public"), ); webapp_client.on("remember_me_failed", () => - redux.getActions("account").set_user_type("public") + redux.getActions("account").set_user_type("public"), ); // Autosave interval diff --git a/src/packages/frontend/account/types.ts b/src/packages/frontend/account/types.ts index 094f482ddf..4a46050b0e 100644 --- a/src/packages/frontend/account/types.ts +++ b/src/packages/frontend/account/types.ts @@ -6,7 +6,6 @@ import { List, Map } from "immutable"; import { TypedMap } from "@cocalc/frontend/app-framework"; -import { MessageInfo } from "@cocalc/frontend/client/hub"; import type { Locale, OTHER_SETTINGS_LOCALE_KEY } from "@cocalc/frontend/i18n"; import { NEW_FILENAMES, @@ -76,7 +75,6 @@ export interface AccountState { reset_key?: string; sign_out_error?: string; show_sign_out?: boolean; - mesg_info?: TypedMap; hub?: string; remember_me?: boolean; has_remember_me?: boolean; @@ -84,7 +82,6 @@ export interface AccountState { is_anonymous: boolean; is_admin: boolean; is_ready: boolean; // user signed in and account settings have been loaded. - doing_anonymous_setup?: boolean; lti_id?: List; created?: Date; strategies?: List>; diff --git a/src/packages/frontend/account/upgrades/project-upgrades-table.tsx b/src/packages/frontend/account/upgrades/project-upgrades-table.tsx index 26aad536ad..b0391e9f59 100644 --- a/src/packages/frontend/account/upgrades/project-upgrades-table.tsx +++ b/src/packages/frontend/account/upgrades/project-upgrades-table.tsx @@ -4,14 +4,12 @@ */ import { Map } from "immutable"; -import { webapp_client } from "../../webapp-client"; import { rclass, redux, rtypes, Component } from "../../app-framework"; -import { ErrorDisplay, UpgradeAdjustor, r_join } from "../../components"; +import { UpgradeAdjustor, r_join } from "../../components"; import { PROJECT_UPGRADES } from "@cocalc/util/schema"; import { ProjectTitle } from "../../projects/project-title"; -import { Button, Row, Col, Panel } from "../../antd-bootstrap"; -import { ResetProjectsConfirmation } from "./reset-projects"; -import { plural, is_string, len, round1 } from "@cocalc/util/misc"; +import { Row, Col, Panel } from "../../antd-bootstrap"; +import { plural, len, round1 } from "@cocalc/util/misc"; interface reduxProps { get_total_upgrades: Function; @@ -26,8 +24,6 @@ interface reduxProps { interface State { show_adjustor: Map; // project_id : bool - expand_remove_all_upgrades: boolean; - remove_all_upgrades_error?: string; } class ProjectUpgradesTable extends Component { @@ -54,8 +50,6 @@ class ProjectUpgradesTable extends Component { super(props, state); this.state = { show_adjustor: Map({}), - expand_remove_all_upgrades: false, - remove_all_upgrades_error: undefined, }; } @@ -98,7 +92,7 @@ class ProjectUpgradesTable extends Component { const info = PROJECT_UPGRADES.params[param]; if (info == null) { console.warn( - `Invalid upgrades database entry for project_id='${project_id}' -- if this problem persists, email ${this.props.help_email} with the project_id: ${param}` + `Invalid upgrades database entry for project_id='${project_id}' -- if this problem persists, email ${this.props.help_email} with the project_id: ${param}`, ); continue; } @@ -106,7 +100,7 @@ class ProjectUpgradesTable extends Component { v.push( {info.display}: {n} {plural(n, info.display_unit)} - + , ); } return r_join(v); @@ -120,7 +114,7 @@ class ProjectUpgradesTable extends Component { upgrades_you_can_use={this.props.get_total_upgrades()} upgrades_you_applied_to_all_projects={this.props.get_total_upgrades_you_have_applied()} upgrades_you_applied_to_this_project={this.props.get_upgrades_you_applied_to_project( - project_id + project_id, )} quota_params={PROJECT_UPGRADES.params} submit_upgrade_quotas={(new_quotas) => @@ -164,58 +158,12 @@ class ProjectUpgradesTable extends Component { const upgrades = upgraded_projects[project_id]; i += 1; result.push( - this.render_upgraded_project(project_id, upgrades, i % 2 === 0) + this.render_upgraded_project(project_id, upgrades, i % 2 === 0), ); } return result; } - async confirm_reset(_e) { - try { - await webapp_client.project_client.remove_all_upgrades(); - } catch (err) { - this.setState({ - expand_remove_all_upgrades: false, - remove_all_upgrades_error: err?.toString(), - }); - } - } - - private render_remove_all_upgrades_error() { - let err: any = this.state.remove_all_upgrades_error; - if (!is_string(err)) { - err = JSON.stringify(err); - } - return ( - - - - this.setState({ remove_all_upgrades_error: undefined }) - } - /> - - - ); - } - - private render_remove_all_upgrades_conf() { - return ( - - - - this.setState({ expand_remove_all_upgrades: false }) - } - /> - - - ); - } - private render_header() { return (
@@ -224,23 +172,8 @@ class ProjectUpgradesTable extends Component {
Upgrades you have applied to projects
- - {this.state.remove_all_upgrades_error - ? this.render_remove_all_upgrades_error() - : undefined} - {this.state.expand_remove_all_upgrades - ? this.render_remove_all_upgrades_conf() - : undefined}
); } diff --git a/src/packages/frontend/admin/site-settings/index.tsx b/src/packages/frontend/admin/site-settings/index.tsx index babe4d562e..66f560889d 100644 --- a/src/packages/frontend/admin/site-settings/index.tsx +++ b/src/packages/frontend/admin/site-settings/index.tsx @@ -13,10 +13,8 @@ import { Modal, Row, } from "antd"; -import { delay } from "awaiting"; import { isEqual } from "lodash"; import { useEffect, useMemo, useRef, useState } from "react"; -import { alert_message } from "@cocalc/frontend/alerts"; import { Well } from "@cocalc/frontend/antd-bootstrap"; import { redux } from "@cocalc/frontend/app-framework"; import useCounter from "@cocalc/frontend/app-framework/counter-hook"; @@ -24,7 +22,7 @@ import { Gap, Icon, Loading, Paragraph } from "@cocalc/frontend/components"; import { query } from "@cocalc/frontend/frame-editors/generic/client"; import { TAGS, Tag } from "@cocalc/util/db-schema/site-defaults"; import { EXTRAS } from "@cocalc/util/db-schema/site-settings-extras"; -import { deep_copy, keys, unreachable } from "@cocalc/util/misc"; +import { deep_copy, keys } from "@cocalc/util/misc"; import { site_settings_conf } from "@cocalc/util/schema"; import { RenderRow } from "./render-row"; import { Data, IsReadonly, State } from "./types"; @@ -39,7 +37,7 @@ const { CheckableTag } = AntdTag; export default function SiteSettings({ close }) { const { inc: change } = useCounter(); const testEmailRef = useRef(null); - const [disableTests, setDisableTests] = useState(false); + const [_, setDisableTests] = useState(false); const [state, setState] = useState("load"); const [error, setError] = useState(""); const [data, setData] = useState(null); @@ -255,59 +253,6 @@ export default function SiteSettings({ close }) { ); } - async function sendTestEmail( - type: "password_reset" | "invite_email" | "mention" | "verification", - ): Promise { - const email = testEmailRef.current?.input?.value; - if (!email) { - alert_message({ - type: "error", - message: "NOT sending test email, since email field is empty", - }); - return; - } - alert_message({ - type: "info", - message: `sending test email "${type}" to ${email}`, - }); - // saving info - await store(); - setDisableTests(true); - // wait 3 secs - await delay(3000); - switch (type) { - case "password_reset": - redux.getActions("account").forgot_password(email); - break; - case "invite_email": - alert_message({ - type: "error", - message: "Simulated invite emails are not implemented yet", - }); - break; - case "mention": - alert_message({ - type: "error", - message: "Simulated mention emails are not implemented yet", - }); - break; - case "verification": - // The code below "looks good" but it doesn't work ??? - // const users = await user_search({ - // query: email, - // admin: true, - // limit: 1 - // }); - // if (users.length == 1) { - // await webapp_client.account_client.send_verification_email(users[0].account_id); - // } - break; - default: - unreachable(type); - } - setDisableTests(false); - } - function Tests() { return (
@@ -320,40 +265,6 @@ export default function SiteSettings({ close }) { defaultValue={redux.getStore("account").get("email_address")} ref={testEmailRef} /> - - { - // commented out since they aren't implemented - // - } - { - // - // - }
); } diff --git a/src/packages/frontend/admin/users/password-reset.tsx b/src/packages/frontend/admin/users/password-reset.tsx index ad87800b0d..f984d4bc50 100644 --- a/src/packages/frontend/admin/users/password-reset.tsx +++ b/src/packages/frontend/admin/users/password-reset.tsx @@ -5,12 +5,17 @@ import { Component, Rendered } from "@cocalc/frontend/app-framework"; import { Button } from "@cocalc/frontend/antd-bootstrap"; -import { CopyToClipBoard, Icon, ErrorDisplay } from "@cocalc/frontend/components"; +import { + CopyToClipBoard, + Icon, + ErrorDisplay, +} from "@cocalc/frontend/components"; import { webapp_client } from "../../webapp-client"; import { appBasePath } from "@cocalc/frontend/customize/app-base-path"; interface Props { - email_address?: string; + account_id: string; + email_address: string; } interface State { @@ -32,12 +37,11 @@ export class PasswordReset extends Component { } async do_request(): Promise { - if (!this.props.email_address) throw Error("bug"); this.setState({ running: true }); let link: string; try { - link = await webapp_client.admin_client.admin_reset_password( - this.props.email_address + link = await webapp_client.conat_client.hub.system.adminResetPasswordLink( + { user_account_id: this.props.account_id }, ); } catch (err) { if (!this.mounted) return; @@ -74,6 +78,7 @@ export class PasswordReset extends Component { } return ( { this.setState({ error: undefined }); diff --git a/src/packages/frontend/admin/users/user.tsx b/src/packages/frontend/admin/users/user.tsx index b1785a61e1..3d902ed09c 100644 --- a/src/packages/frontend/admin/users/user.tsx +++ b/src/packages/frontend/admin/users/user.tsx @@ -155,9 +155,12 @@ export function UserResult({ last_name={last_name ?? ""} /> )} - {state.password && ( + {state.password && email_address && ( - + )} {state.ban && ( diff --git a/src/packages/frontend/alerts.ts b/src/packages/frontend/alerts.ts index 9b95182850..6280007a83 100644 --- a/src/packages/frontend/alerts.ts +++ b/src/packages/frontend/alerts.ts @@ -18,10 +18,10 @@ import { webapp_client } from "./webapp-client"; type NotificationType = "error" | "default" | "success" | "info" | "warning"; const default_timeout: { [key: string]: number } = { - error: 8, - default: 4, - success: 4, - info: 6, + error: 9, + default: 6, + success: 5, + info: 7, }; const last_shown = {}; @@ -72,7 +72,7 @@ export function alert_message(opts: AlertMessageOptions = {}) { } f({ message: opts.title != null ? opts.title : "", - description: opts.message, + description: stripExcessiveError(opts.message), duration: opts.block ? 0 : opts.timeout, }); @@ -85,25 +85,6 @@ export function alert_message(opts: AlertMessageOptions = {}) { } } -function check_for_clock_skew() { - const local_time = Date.now(); - const s = Math.ceil( - Math.abs( - webapp_client.time_client.server_time().valueOf() - local_time.valueOf() - ) / 1000 - ); - if (s > 120) { - return alert_message({ - type: "error", - timeout: 9999, - message: `Your computer's clock is off by about ${s} seconds! You MUST set it correctly then refresh your browser. Expect nothing to work until you fix this.`, - }); - } -} - -// Wait until after the page is loaded and clock sync'd before checking for skew. -setTimeout(check_for_clock_skew, 60000); - // for testing/development /* alert_message({ type: "error", message: "This is an error" }); @@ -112,3 +93,14 @@ alert_message({ type: "warning", message: "This is a warning alert" }); alert_message({ type: "success", message: "This is a success alert" }); alert_message({ type: "info", message: "This is an info alert" }); */ + +function stripExcessiveError(s) { + if (typeof s != "string") { + return s; + } + s = s.trim(); + if (s.startsWith("Error: Error:")) { + s = s.slice("Error: ".length); + } + return s; +} diff --git a/src/packages/frontend/app-framework/Table.ts b/src/packages/frontend/app-framework/Table.ts index caaeab2437..59fac3bb87 100644 --- a/src/packages/frontend/app-framework/Table.ts +++ b/src/packages/frontend/app-framework/Table.ts @@ -7,8 +7,6 @@ import { AppRedux } from "../app-framework"; import { bind_methods } from "@cocalc/util/misc"; import { webapp_client } from "../webapp-client"; -declare let Primus; - export type TableConstructor = new (name, redux) => T; export abstract class Table { diff --git a/src/packages/frontend/app-framework/__tests__/store-get-in.type.test.ts b/src/packages/frontend/app-framework/__tests__/store-get-in.type.test.ts deleted file mode 100644 index 03ab5d1b56..0000000000 --- a/src/packages/frontend/app-framework/__tests__/store-get-in.type.test.ts +++ /dev/null @@ -1,50 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -import { expectType } from "tsd"; - -import { AppRedux, Store } from "@cocalc/frontend/app-framework"; -import { DeepImmutable } from "@cocalc/util/types/immutable-types"; - -test("Mapping with maybes in state", () => { - interface State { - deep?: { values: { cake: string } }; - } - const redux = new AppRedux(); - - const one_maybes = new Store("", redux); - - let withMaybeValues1 = one_maybes.getIn(["deep"]); - withMaybeValues1 = undefined; // Expect Assignable - expectType>( - withMaybeValues1, - ); - - let withMaybeValues2 = one_maybes.getIn(["deep", "values"]); - withMaybeValues2 = undefined; // Expect Assignable - expectType>(withMaybeValues2); - - let withMaybeValues3 = one_maybes.getIn(["deep", "values", "cake"]); - withMaybeValues3 = undefined; // Expect Assignable - expectType(withMaybeValues3); -}); - -test("Mapping with no maybes in State", () => { - interface State { - deep: { values: { cake: string } }; - } - - const redux = new AppRedux(); - const no_maybes = new Store("", redux); - - const value1 = no_maybes.getIn(["deep"]); - expectType>(value1); - - const value2 = no_maybes.getIn(["deep", "values"]); - expectType>(value2); - - const value3 = no_maybes.getIn(["deep", "values", "cake"]); - expectType(value3); -}); diff --git a/src/packages/frontend/app-framework/index.ts b/src/packages/frontend/app-framework/index.ts index c49c477ed4..19b1acaf36 100644 --- a/src/packages/frontend/app-framework/index.ts +++ b/src/packages/frontend/app-framework/index.ts @@ -210,9 +210,7 @@ export class AppRedux extends AppReduxBase { } const name = project_redux_name(project_id); const store = this.getStore(name); - if (store && typeof store.destroy == "function") { - store.destroy(); - } + store?.destroy?.(); this.removeActions(name); this.removeStore(name); } diff --git a/src/packages/frontend/app/actions.ts b/src/packages/frontend/app/actions.ts index c4e618782a..1624d8053a 100644 --- a/src/packages/frontend/app/actions.ts +++ b/src/packages/frontend/app/actions.ts @@ -264,11 +264,9 @@ export class PageActions extends Actions { this.setState({ ping, avgping }); } - set_connection_status(connection_status, time: Date) { - if (time > (redux.getStore("page").get("last_status_time") ?? 0)) { - this.setState({ connection_status, last_status_time: time }); - } - } + set_connection_status = (connection_status, time: Date) => { + this.setState({ connection_status, last_status_time: time }); + }; set_connection_quality(connection_quality) { this.setState({ connection_quality }); diff --git a/src/packages/frontend/app/connection-indicator.tsx b/src/packages/frontend/app/connection-indicator.tsx index 5a8360d374..979e64e8d2 100644 --- a/src/packages/frontend/app/connection-indicator.tsx +++ b/src/packages/frontend/app/connection-indicator.tsx @@ -42,11 +42,10 @@ export const ConnectionIndicator: React.FC = React.memo( const intl = useIntl(); const hub_status = useTypedRedux("page", "connection_status"); - const mesg_info = useTypedRedux("account", "mesg_info"); const actions = useActions("page"); - const nats = useTypedRedux("page", "nats"); - const nats_status = nats?.get("state") ?? "disconnected"; - const connection_status = worst(hub_status, nats_status); + const conat = useTypedRedux("page", "conat"); + const conatState = conat?.get("state") ?? "disconnected"; + const connection_status = worst(hub_status, conatState); const connecting_style: CSS = { flex: "1", @@ -75,18 +74,12 @@ export const ConnectionIndicator: React.FC = React.memo( function render_connection_status() { if (connection_status === "connected") { - const icon_style: CSS = { ...BASE_STYLE, fontSize: fontSizeIcons }; - if (mesg_info?.get("enqueued") ?? 0 > 6) { - // serious backlog of data! - icon_style.color = "red"; - } else if (mesg_info?.get("count") ?? 0 > 2) { - // worrisome amount - icon_style.color = "#08e"; - } else if (mesg_info?.get("count") ?? 0 > 0) { - // working well but doing something minimal - icon_style.color = "#00c"; - } - return ; + return ( + + ); } else if (connection_status === "connecting") { return (
diff --git a/src/packages/frontend/app/connection-info.tsx b/src/packages/frontend/app/connection-info.tsx index 5276951bef..2b2c3dd72b 100644 --- a/src/packages/frontend/app/connection-info.tsx +++ b/src/packages/frontend/app/connection-info.tsx @@ -5,7 +5,6 @@ import { Modal } from "antd"; import { FormattedMessage, useIntl } from "react-intl"; -import { A } from "@cocalc/frontend/components/A"; import { Button, Col, Row } from "@cocalc/frontend/antd-bootstrap"; import { React, @@ -15,7 +14,7 @@ import { import { Icon } from "@cocalc/frontend/components"; import { labels } from "@cocalc/frontend/i18n"; import { webapp_client } from "@cocalc/frontend/webapp-client"; -import { COLORS } from "@cocalc/util/theme"; +import { ConnectionStatsDisplay } from "./connection-status"; export const ConnectionInfo: React.FC = React.memo(() => { const intl = useIntl(); @@ -23,9 +22,9 @@ export const ConnectionInfo: React.FC = React.memo(() => { const ping = useTypedRedux("page", "ping"); const avgping = useTypedRedux("page", "avgping"); const status = useTypedRedux("page", "connection_status"); - const hub = useTypedRedux("account", "hub"); const page_actions = useActions("page"); - const nats = useTypedRedux("page", "nats"); + const conat = useTypedRedux("page", "conat"); + const hub = useTypedRedux("account", "hub"); function close() { page_actions.show_connection(false); @@ -33,7 +32,7 @@ export const ConnectionInfo: React.FC = React.memo(() => { return ( { {" "} {intl.formatMessage(labels.connection)}
- @@ -52,18 +55,27 @@ export const ConnectionInfo: React.FC = React.memo(() => { } >
- {ping ? ( + {conat != null && ( + + {conat && ( + + )} + + + )} + {ping ? ( + -

+

-
+ - + 
                  {
             
           
         ) : undefined}
-        
-          
-            

-              NATS client
-            

-          
-          {nats != null && (
-            
-              
-                {JSON.stringify(nats.toJS(), undefined, 2)
-                  .replace(/{|}|,|\"/g, "")
-                  .trim()
-                  .replace("  data:", "data:")}
-              

-            
-          )}
-        
-        
-          
-            

-              
-            

-          
-          
-            
{hub != null ? hub : "Not signed in"}

-          
-        
-        
-          
-            

-              Hub {intl.formatMessage(labels.message_plural, { num: 10 })}
-            

-          
-          
-            
-          
-
); }); - -function bytes_to_str(bytes: number): string { - const x = Math.round(bytes / 1000); - if (x < 1000) { - return x + "K"; - } - return x / 1000 + "M"; -} - -const MessageInfo: React.FC = React.memo(() => { - const intl = useIntl(); - - const info = useTypedRedux("account", "mesg_info"); - - if (info == null) { - return ; - } - - function messages(num: number): string { - return `${num} ${intl.formatMessage(labels.message_plural, { num })}`; - } - - const sent = intl.formatMessage({ - id: "connection-info.messages_sent", - defaultMessage: "sent", - description: "Messages sent", - }); - - const received = intl.formatMessage({ - id: "connection-info.messages_received", - defaultMessage: "received", - description: "Messages received", - }); - - return ( -
- 
-        {messages(info.get("sent"))} {sent} (
-        {bytes_to_str(info.get("sent_length"))})
-        

-        {messages(info.get("recv"))} {received} (
-        {bytes_to_str(info.get("recv_length"))})
-        

-         0
-              ? { color: "#08e", fontWeight: "bold" }
-              : undefined
-          }
-        >
-          {messages(info.get("count"))} in flight
-        
-        

-        {messages(info.get("enqueued"))} queued to send
-
-
- -
-
- ); -}); diff --git a/src/packages/frontend/app/connection-status.tsx b/src/packages/frontend/app/connection-status.tsx new file mode 100644 index 0000000000..d7bd24a4b0 --- /dev/null +++ b/src/packages/frontend/app/connection-status.tsx @@ -0,0 +1,177 @@ +import { Badge, Progress, Descriptions, Typography, Space, Alert } from "antd"; +import { + CheckCircleOutlined, + CloseCircleOutlined, + SendOutlined, + DownloadOutlined, + UsergroupAddOutlined, + DatabaseOutlined, +} from "@ant-design/icons"; +import type { ConatConnectionStatus } from "@cocalc/frontend/conat/client"; +import { capitalize } from "@cocalc/util/misc"; +import { MAX_SUBSCRIPTIONS_PER_CLIENT } from "@cocalc/conat/core/constants"; + +let MAX_SEND_MESSAGES = 1000, + MAX_SEND_BYTES = 1_000_000; +let MAX_RECV_MESSAGES = 2000, + MAX_RECV_BYTES = 10_000_000; + +function bytesToStr(bytes: number): string { + if (bytes < 1024) return bytes + " B"; + if (bytes < 1024 * 1024) return (bytes / 1024).toFixed(1) + " KB"; + return (bytes / (1024 * 1024)).toFixed(2) + " MB"; +} + +export function ConnectionStatsDisplay({ + status, + hub, +}: { + status: ConatConnectionStatus; + hub?: string; +}) { + const connected = status.state === "connected"; + const statusText = connected + ? `Connected${hub ? " to hub " + hub : ""}` + : "Disconnected"; + const statusColor = connected ? "green" : "red"; + + const icon = connected ? : ; + + if (MAX_SEND_MESSAGES <= status.stats.send.messages) { + MAX_SEND_MESSAGES *= 1.2; + } + if (MAX_SEND_BYTES <= status.stats.send.bytes) { + MAX_SEND_BYTES *= 1.2; + } + if (MAX_RECV_MESSAGES <= status.stats.recv.messages) { + MAX_RECV_MESSAGES *= 1.2; + } + if (MAX_RECV_BYTES <= status.stats.recv.bytes) { + MAX_RECV_BYTES *= 1.2; + } + + if (status?.stats == null) { + return null; + } + + return ( + + + {icon} + + + {!connected && !!status.reason && ( + + )} + + + + Messages sent + + } + > + `${status.stats.send.messages}`} + /> + + + Bytes sent + + } + > + bytesToStr(status.stats.send.bytes)} + /> + + + Messages received + + } + > + `${status.stats.recv.messages}`} + /> + + + Bytes received + + } + > + bytesToStr(status.stats.recv.bytes)} + /> + + + Subscriptions + + } + > + `${status.stats.subs}`} + /> + + + + {/* Optionally, details debugging */} + {status.details && ( + + {JSON.stringify(status.details, null, 2)} + + )} + + ); +} diff --git a/src/packages/frontend/app/monitor-connection.ts b/src/packages/frontend/app/monitor-connection.ts index f777f11ac9..25f239f728 100644 --- a/src/packages/frontend/app/monitor-connection.ts +++ b/src/packages/frontend/app/monitor-connection.ts @@ -105,7 +105,7 @@ export function init_connection(): void { actions.set_connection_status("connecting", date); } - const attempt = webapp_client.hub_client.get_num_attempts(); + const attempt = webapp_client.conat_client.numConnectionAttempts; async function reconnect(msg) { // reset recent disconnects, and hope that after the reconnection the situation will be better recent_disconnects.length = 0; // see https://stackoverflow.com/questions/1232040/how-do-i-empty-an-array-in-javascript @@ -116,7 +116,7 @@ export function init_connection(): void { if (!recent_wakeup_from_standby()) { alert_message(msg); } - webapp_client.hub_client.fix_connection(); + webapp_client.conat_client.reconnect(); // Wait a half second, then remove one extra reconnect added by the call in the above line. await delay(500); recent_disconnects.pop(); diff --git a/src/packages/frontend/app/page.tsx b/src/packages/frontend/app/page.tsx index 94ae019022..4bfe138525 100644 --- a/src/packages/frontend/app/page.tsx +++ b/src/packages/frontend/app/page.tsx @@ -23,7 +23,6 @@ import { useTypedRedux, } from "@cocalc/frontend/app-framework"; import { IconName, Icon } from "@cocalc/frontend/components/icon"; -import { SiteName } from "@cocalc/frontend/customize"; import { FileUsePage } from "@cocalc/frontend/file-use/page"; import { labels } from "@cocalc/frontend/i18n"; import { ProjectsNav } from "@cocalc/frontend/projects/projects-nav"; @@ -47,7 +46,8 @@ import PopconfirmModal from "./popconfirm-modal"; import SettingsModal from "./settings-modal"; import { HIDE_LABEL_THRESHOLD, NAV_CLASS } from "./top-nav-consts"; import { useShowVerifyEmail, VerifyEmail } from "./verify-email-banner"; -import { CookieWarning, LocalStorageWarning, VersionWarning } from "./warnings"; +import { CookieWarning, LocalStorageWarning } from "./warnings"; +import VersionWarning from "./version-warning"; import Next from "@cocalc/frontend/components/next"; import { ClientContext } from "@cocalc/frontend/client/context"; import { webapp_client } from "@cocalc/frontend/webapp-client"; @@ -98,6 +98,11 @@ export const Page: React.FC = () => { }; }, []); + const [showSignInTab, setShowSignInTab] = useState(false); + useEffect(() => { + setTimeout(() => setShowSignInTab(true), 3000); + }, []); + const active_top_tab = useTypedRedux("page", "active_top_tab"); const show_mentions = active_top_tab === "notifications"; const show_connection = useTypedRedux("page", "show_connection"); @@ -105,16 +110,11 @@ export const Page: React.FC = () => { const fullscreen = useTypedRedux("page", "fullscreen"); const local_storage_warning = useTypedRedux("page", "local_storage_warning"); const cookie_warning = useTypedRedux("page", "cookie_warning"); - const new_version = useTypedRedux("page", "new_version"); const accountIsReady = useTypedRedux("account", "is_ready"); const account_id = useTypedRedux("account", "account_id"); const is_logged_in = useTypedRedux("account", "is_logged_in"); const is_anonymous = useTypedRedux("account", "is_anonymous"); - const doing_anonymous_setup = useTypedRedux( - "account", - "doing_anonymous_setup", - ); const when_account_created = useTypedRedux("account", "created"); const groups = useTypedRedux("account", "groups"); const show_verify_email: boolean = useShowVerifyEmail(); @@ -213,7 +213,7 @@ export const Page: React.FC = () => { } function render_sign_in_tab(): JSX.Element | null { - if (is_logged_in) return null; + if (is_logged_in || !showSignInTab) return null; return ( { } } - let body; - if (doing_anonymous_setup) { - // Don't show the login screen or top navbar for a second - // while creating their anonymous account, since that - // would just be ugly/confusing/and annoying. - // Have to use above style to *hide* the crash warning. - const loading_anon = ( -
-

- -

-
- Please give a couple of seconds to start your project and - prepare a file... + // Children must define their own padding from navbar and screen borders + // Note that the parent is a flex container + const body = ( +
e.preventDefault()} + onDrop={drop} + > + {insecure_test_mode && } + {show_file_use && ( +
+
-
- ); - body =
{loading_anon}
; - } else { - // Children must define their own padding from navbar and screen borders - // Note that the parent is a flex container - body = ( -
e.preventDefault()} - onDrop={drop} - > - {insecure_test_mode && } - {show_file_use && ( -
- -
- )} - {show_connection && } - {new_version && } - {cookie_warning && } - {local_storage_warning && } - {show_i18n && } - {show_verify_email && } - {!fullscreen && ( - - )} - {fullscreen && render_fullscreen()} - {isNarrow && ( - - )} - - - - -
- ); - return ( - - {body} - - ); - } + )} + {show_connection && } + + {cookie_warning && } + {local_storage_warning && } + {show_i18n && } + {show_verify_email && } + {!fullscreen && ( + + )} + {fullscreen && render_fullscreen()} + {isNarrow && ( + + )} + + + + +
+ ); + return ( + + {body} + + ); }; diff --git a/src/packages/frontend/app/store.ts b/src/packages/frontend/app/store.ts index ffa1a3e131..0157c3bba7 100644 --- a/src/packages/frontend/app/store.ts +++ b/src/packages/frontend/app/store.ts @@ -6,6 +6,7 @@ import { redux, Store, TypedMap } from "@cocalc/frontend/app-framework"; import target from "@cocalc/frontend/client/handle-target"; import { parse_target } from "../history"; +import type { ConatConnectionStatus } from "@cocalc/frontend/conat/client"; type TopTab = | "about" // the "/help" page @@ -52,16 +53,7 @@ export interface PageState { }; settingsModal?: string; - nats?: TypedMap<{ - state: ConnectionStatus; - data: { - inBytes?: number; - inMsgs?: number; - outBytes?: number; - outMsgs?: number; - }; - numConnections: number; - }>; + conat?: TypedMap; } export class PageStore extends Store {} diff --git a/src/packages/frontend/app/version-warning.tsx b/src/packages/frontend/app/version-warning.tsx new file mode 100644 index 0000000000..350423285b --- /dev/null +++ b/src/packages/frontend/app/version-warning.tsx @@ -0,0 +1,120 @@ +/* + * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. + * License: MS-RSL – see LICENSE.md for details + */ + +import { useTypedRedux } from "@cocalc/frontend/app-framework"; +import { Gap, Icon } from "@cocalc/frontend/components"; +import { type CSSProperties, useEffect, useState } from "react"; +import { version } from "@cocalc/util/smc-version"; +import { webapp_client } from "@cocalc/frontend/webapp-client"; + +const VERSION_WARNING_STYLE = { + fontSize: "12pt", + position: "fixed", + left: 12, + backgroundColor: "#fcf8e3", + color: "#8a6d3b", + top: 20, + borderRadius: 4, + padding: "15px", + zIndex: 900, + boxShadow: "8px 8px 4px #888", + width: "70%", + marginTop: "1em", +} as CSSProperties; + +export default function VersionWarning() { + const [closed, setClosed] = useState(false); + const minVersion = useTypedRedux("customize", "version_min_browser"); + const recommendedVersion = useTypedRedux( + "customize", + "version_recommended_browser", + ); + + useEffect(() => { + if (minVersion > version) { + // immediately and permenantly disconnect user from conat + webapp_client.conat_client.permanentlyDisconnect(); + } + }, [minVersion]); + + if (version >= recommendedVersion) { + return null; + } + + if (version >= minVersion && closed) { + return null; + } + + const style = { + ...VERSION_WARNING_STYLE, + ...(version < minVersion + ? { backgroundColor: "red", color: "#fff" } + : undefined), + }; + + function render_critical() { + if (version >= minVersion) { + return; + } + return ( + + ); + } + + function render_suggested() { + return ( + <> + New Version Available: upgrade by + window.location.reload()} + style={{ + cursor: "pointer", + fontWeight: "bold", + color: style.color, + textDecoration: "underline", + }} + > + reloading this page + + .{render_close()} + + ); + } + + function render_close() { + if (version >= minVersion) { + return ( + setClosed(true)} + /> + ); + } + } + + return ( +
+ {render_suggested()} + {render_critical()} +
+ ); +} diff --git a/src/packages/frontend/app/warnings.tsx b/src/packages/frontend/app/warnings.tsx index 4ee0411c19..c11a7cbc44 100644 --- a/src/packages/frontend/app/warnings.tsx +++ b/src/packages/frontend/app/warnings.tsx @@ -3,97 +3,12 @@ * License: MS-RSL – see LICENSE.md for details */ -import { React, redux, TypedMap } from "@cocalc/frontend/app-framework"; -import { Gap, Icon } from "@cocalc/frontend/components"; +import { Icon } from "@cocalc/frontend/components"; import { SiteName } from "@cocalc/frontend/customize"; import { get_browser } from "@cocalc/frontend/feature"; -import { webapp_client } from "@cocalc/frontend/webapp-client"; +import { type CSSProperties } from "react"; -interface VersionWarningProps { - new_version: TypedMap<{ min_version: number; version: number }>; -} - -const VERSION_WARNING_STYLE: React.CSSProperties = { - fontSize: "12pt", - position: "fixed", - left: 12, - backgroundColor: "#fcf8e3", - color: "#8a6d3b", - top: 20, - borderRadius: 4, - padding: "15px", - zIndex: 900, - boxShadow: "8px 8px 4px #888", - width: "70%", - marginTop: "1em", -} as const; - -export const VersionWarning: React.FC = React.memo( - ({ new_version }) => { - function render_critical() { - if (new_version.get("min_version") <= webapp_client.version()) { - return; - } - return ( - - ); - } - - function render_close() { - if (new_version.get("min_version") <= webapp_client.version()) { - return ( - redux.getActions("page").set_new_version(undefined)} - /> - ); - } - } - - let style: React.CSSProperties = VERSION_WARNING_STYLE; - if (new_version.get("min_version") > webapp_client.version()) { - style = { ...style, ...{ backgroundColor: "red", color: "#fff" } }; - } - - return ( - - ); - }, -); - -const WARNING_STYLE: React.CSSProperties = { +const WARNING_STYLE = { position: "fixed", left: 12, backgroundColor: "red", @@ -106,23 +21,23 @@ const WARNING_STYLE: React.CSSProperties = { zIndex: 100000, boxShadow: "8px 8px 4px #888", width: "70%", -} as const; +} as CSSProperties; -export const CookieWarning: React.FC = React.memo(() => { +export function CookieWarning() { return (
You must enable cookies to use{" "} .
); -}); +} -const STORAGE_WARNING_STYLE: React.CSSProperties = { +const STORAGE_WARNING_STYLE = { ...WARNING_STYLE, top: 55, -}; +} as CSSProperties; -export const LocalStorageWarning: React.FC = React.memo(() => { +export function LocalStorageWarning() { return (
You must enable local storage to use{" "} @@ -133,4 +48,4 @@ export const LocalStorageWarning: React.FC = React.memo(() => { .
); -}); +} diff --git a/src/packages/frontend/billing/actions.ts b/src/packages/frontend/billing/actions.ts index 2751c309e8..1ad5c363cc 100644 --- a/src/packages/frontend/billing/actions.ts +++ b/src/packages/frontend/billing/actions.ts @@ -5,24 +5,19 @@ // COMPLETEY DEPRECATED -- DELETE THIS ? - /* Billing actions. These are mainly for interfacing with Stripe. They are all async (no callbacks!). + +**PRETTY MUCH DEPRECATED** */ import { fromJS, Map } from "immutable"; import { redux, Actions, Store } from "../app-framework"; import { reuse_in_flight_methods } from "@cocalc/util/async-utils"; -import { - server_minutes_ago, - server_time, - server_days_ago, -} from "@cocalc/util/misc"; -import { webapp_client } from "../webapp-client"; -import { StripeClient } from "../client/stripe"; +import { server_days_ago } from "@cocalc/util/misc"; import { getManagedLicenses } from "../account/licenses/util"; import { BillingStoreState } from "./store"; @@ -31,15 +26,12 @@ require("./store"); // ensure 'billing' store is created so can set this.store b export class BillingActions extends Actions { private store: Store; - private last_subscription_attempt?: any; - private stripe: StripeClient; constructor(name: string, redux: any) { super(name, redux); const store = redux.getStore("billing"); if (store == null) throw Error("bug -- billing store should be defined"); this.store = store; - this.stripe = webapp_client.stripe; reuse_in_flight_methods(this, ["update_customer"]); } @@ -48,151 +40,13 @@ export class BillingActions extends Actions { } public async update_customer(): Promise { - const is_commercial = redux - .getStore("customize") - .get("is_commercial", false); - if (!is_commercial) { - return; - } - this.setState({ action: "Updating billing information" }); - try { - const resp = await this.stripe.get_customer(); - if (!resp.stripe_publishable_key) { - this.setState({ no_stripe: true }); - throw Error( - "WARNING: Stripe is not configured -- billing not available", - ); - } - this.setState({ - customer: resp.customer, - loaded: true, - stripe_publishable_key: resp.stripe_publishable_key, - }); - if (resp.customer) { - // TODO: only call get_invoices if the customer already exists in the system! - // FUTURE: -- this {limit:100} will change when we use webhooks and our own database of info... - const invoices = await this.stripe.get_invoices({ - limit: 100, - }); - this.setState({ invoices }); - } - } catch (err) { - this.setState({ error: err }); - throw err; - } finally { - this.setState({ action: "" }); - } - } - - // Call a webapp_client.stripe. function with given opts, returning - // the result (which matters only for coupons?). - // This is wrapped as an async call, and also sets the action and error - // states of the Store so the UI can reflect what is happening. - // Also, after update_customer gets called, to update the UI. - // If there is an error, this also throws that error (so it is NOT just - // reflected in the UI). - private async stripe_action( - f: Function, - desc: string, - ...args - ): Promise { - this.setState({ action: desc }); - try { - return await f.bind(this.stripe)(...args); - } catch (err) { - this.setState({ error: `${err}` }); - throw err; - } finally { - this.setState({ action: "" }); - await this.update_customer(); - } + return; } public clear_action(): void { this.setState({ action: "", error: "" }); } - public async delete_payment_method(card_id: string): Promise { - await this.stripe_action( - this.stripe.delete_source, - "Deleting a payment method", - card_id, - ); - } - - public async set_as_default_payment_method(card_id: string): Promise { - await this.stripe_action( - this.stripe.set_default_source, - "Setting payment method as default", - card_id, - ); - } - - public async submit_payment_method(token: string): Promise { - await this.stripe_action( - this.stripe.create_source, - "Creating a new payment method (sending token)", - token, - ); - } - - public async cancel_subscription(subscription_id: string): Promise { - await this.stripe_action( - this.stripe.cancel_subscription, - "Cancel a subscription", - { subscription_id }, - ); - } - - public async create_subscription(plan: string): Promise { - const lsa = this.last_subscription_attempt; - if ( - lsa != null && - lsa.plan == plan && - lsa.timestamp > server_minutes_ago(2) - ) { - this.setState({ - action: "", - error: - "Too many subscription attempts in the last minute. Please **REFRESH YOUR BROWSER** THEN DOUBLE CHECK YOUR SUBSCRIPTION LIST.", - }); - return; - } - let coupon: any; - this.setState({ error: "" }); - const applied_coupons = this.store.get("applied_coupons"); - if (applied_coupons != null && applied_coupons.size > 0) { - coupon = applied_coupons.first(); - } - const opts = { - plan, - coupon_id: coupon?.id, - }; - await this.stripe_action( - this.stripe.create_subscription, - "Create a subscription", - opts, - ); - this.last_subscription_attempt = { timestamp: server_time(), plan }; - } - - public async apply_coupon(coupon_id: string): Promise { - try { - const coupon = await this.stripe_action( - this.stripe.get_coupon, - `Applying coupon: ${coupon_id}`, - coupon_id, - ); - const applied_coupons = this.store - .get("applied_coupons", Map()) - .set(coupon.id, coupon); - if (applied_coupons == null) throw Error("BUG -- can't happen"); - this.setState({ applied_coupons, coupon_error: "" }); - } catch (err) { - return this.setState({ coupon_error: `${err}` }); - } - } - public clear_coupon_error(): void { this.setState({ coupon_error: "" }); } @@ -209,31 +63,6 @@ export class BillingActions extends Actions { }); } - // Cancel all subscriptions, remove credit cards, etc. -- this is not a normal action, - // and is used only when deleting an account. - public async cancel_everything(): Promise { - // update info about this customer - await this.update_customer(); - // delete stuff - // delete payment methods - const payment_methods = this.store.getIn(["customer", "sources", "data"]); - if (payment_methods != null) { - for (const x of payment_methods.toJS() as any) { - await this.delete_payment_method(x.id); - } - } - const subscriptions = this.store.getIn([ - "customer", - "subscriptions", - "data", - ]); - if (subscriptions != null) { - for (const x of subscriptions.toJS() as any) { - await this.cancel_subscription(x.id); - } - } - } - // Set this while we are paying for the course. public set_is_paying_for_course( project_id: string, diff --git a/src/packages/frontend/billing/add-payment-method.tsx b/src/packages/frontend/billing/add-payment-method.tsx deleted file mode 100644 index 79d042ff09..0000000000 --- a/src/packages/frontend/billing/add-payment-method.tsx +++ /dev/null @@ -1,174 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -import { Button, ButtonToolbar, Row, Col, Well } from "../antd-bootstrap"; -import { Component, Rendered, redux } from "../app-framework"; -import { ErrorDisplay, Loading } from "../components"; -import { HelpEmailLink } from "../customize"; -import { powered_by_stripe } from "./util"; -import { loadStripe, StripeCard } from "./stripe"; - -interface Props { - on_close?: Function; // optionally called when this should be closed - hide_cancel_button?: boolean; -} - -const CARD_STYLE = { - margin: "15px", - border: "1px solid grey", - padding: "30px", - background: "white", - borderRadius: "5px", -}; - -interface State { - submitting: boolean; - error: string; - loading: boolean; -} - -export class AddPaymentMethod extends Component { - private mounted: boolean = false; - private card?: StripeCard; - - constructor(props, state) { - super(props, state); - this.state = { - submitting: false, - error: "", - loading: true, - }; - } - - public async componentDidMount(): Promise { - this.mounted = true; - const stripe = await loadStripe(); - if (!this.mounted) return; - this.setState({ loading: false }); - const elements = stripe.elements(); - this.card = elements.create("card"); - if (this.card == null) throw Error("bug -- card cannot be null"); - this.card.mount("#card-element"); - } - - public componentWillUnmount(): void { - this.mounted = false; - } - - private async submit_payment_method(): Promise { - this.setState({ error: "", submitting: true }); - const actions = redux.getActions("billing"); - const store = redux.getStore("billing"); - if (store.get("customer") == null) { - actions.setState({ continue_first_purchase: true }); - } - const stripe = await loadStripe(); - let result: { - error?: { message: string }; - token?: { id: string }; - } = {}; - try { - // @ts-ignore - result = await stripe.createToken(this.card); - if (!this.mounted) return; - if (result.error != null) { - this.setState({ error: result.error.message }); - return; - } else if (result.token != null) { - await actions.submit_payment_method(result.token.id); - if (!this.mounted) return; - } - } catch (err) { - if (this.mounted) { - result.error = { message: err.toString() }; // used in finally - this.setState({ error: err.toString() }); - } - } finally { - if (this.mounted) { - this.setState({ submitting: false }); - if (this.props.on_close != null && result.error == null) - this.props.on_close(); - } - } - } - - private render_cancel_button(): Rendered { - if (this.props.hide_cancel_button) return; - return ( - - ); - } - - private render_add_button(): Rendered { - return ( - - ); - } - - private render_payment_method_buttons(): Rendered { - return ( -
- - {powered_by_stripe()} - - - {this.render_add_button()} - {this.render_cancel_button()} - - - -
- (Wire transfers for non-recurring purchases above $100 are possible. - Please email - .) -
-
- ); - } - - private render_error(): Rendered { - if (this.state.error) { - return ( - this.setState({ error: "" })} - /> - ); - } - } - - private render_card(): Rendered { - return ( -
- {this.state.loading ? : undefined} -
- {/* a Stripe Element will be inserted here. */} -
-
- ); - } - - public render(): Rendered { - return ( - - {this.render_card()} - {this.render_error()} - {this.render_payment_method_buttons()} - - ); - } -} diff --git a/src/packages/frontend/billing/billing-page-link.tsx b/src/packages/frontend/billing/billing-page-link.tsx deleted file mode 100644 index 4b0b07b44f..0000000000 --- a/src/packages/frontend/billing/billing-page-link.tsx +++ /dev/null @@ -1,23 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -import { Rendered } from "../app-framework"; -import { load_target } from "../history"; - -export function BillingPageLink(opts: { text?: string }): Rendered { - let { text } = opts; - if (!text) { - text = "billing page"; - } - return ( - - {text} - - ); -} - -export function visit_billing_page(): void { - load_target("settings/billing"); -} diff --git a/src/packages/frontend/billing/billing-page.tsx b/src/packages/frontend/billing/billing-page.tsx deleted file mode 100644 index d45676f7bb..0000000000 --- a/src/packages/frontend/billing/billing-page.tsx +++ /dev/null @@ -1,208 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -// Ensure the billing Actions and Store are created: -import "./actions"; - -import { - Component, - rclass, - redux, - Rendered, - rtypes, -} from "@cocalc/frontend/app-framework"; -import { - A, - ActivityDisplay, - ErrorDisplay, - Loading, -} from "@cocalc/frontend/components"; -import { - Footer, - HelpEmailLink, - PolicyPricingPageUrl, -} from "@cocalc/frontend/customize"; -import { Map } from "immutable"; -import { PaymentMethods } from "./payment-methods"; -import { Customer, InvoicesMap } from "./types"; - -interface ReactProps { - is_simplified?: boolean; - for_course?: boolean; -} - -interface ReduxProps { - customer?: Customer; - invoices?: InvoicesMap; - error?: string | Error; - action?: string; - loaded?: boolean; - no_stripe?: boolean; // if true, stripe definitely isn't configured on the server - selected_plan: string; - project_map: Map; // used, e.g., for course project payments; also computing available upgrades - stripe_customer: Map; // to get total upgrades user has available -} - -export const BillingPage = rclass( - class BillingPage extends Component { - static reduxProps() { - return { - billing: { - customer: rtypes.object, - invoices: rtypes.immutable.Map, - error: rtypes.oneOfType([rtypes.string, rtypes.object]), - action: rtypes.string, - loaded: rtypes.bool, - no_stripe: rtypes.bool, // if true, stripe definitely isn't configured on the server - selected_plan: rtypes.string, - }, - projects: { - project_map: rtypes.immutable, // used, e.g., for course project payments; also computing available upgrades - }, - account: { - stripe_customer: rtypes.immutable, // to get total upgrades user has available - }, - }; - } - - private render_action(): Rendered { - if (this.props.action) { - return ( - redux.getActions("billing").clear_action()} - /> - ); - } - } - - private render_error(): Rendered { - if (this.props.error) { - return ( - redux.getActions("billing").clear_error()} - /> - ); - } - } - - private render_enterprise_support(): Rendered { - return ( -
  • - Enterprise Support: Contact us at for{" "} - enterprise support, including customized course packages, - modified terms of service, additional legal agreements, purchase - orders, insurance and priority technical support. -
    • - ); - } - - private render_on_prem(): Rendered { - return ( -
  • - Commercial on Premises: Contact us at for{" "} - questions about our{" "} - - commercial on premises offering. - -
    • - ); - } - - private render_help_suggestion(): Rendered { - return ( - <> -
  • - Questions: - If you have any questions at all, read the{" "} - - Billing{"/"}Upgrades FAQ - {" "} - or email . -
    • - -
  • - Teaching:{" "} - Contact us} /> if you are - considering purchasing a course subscription and need a short - evaluation trial. -
    • - {this.render_enterprise_support()} - {this.render_on_prem()} - - ); - } - - private counts(): { cards: number; subs: number; invoices: number } { - const cards = this.props.customer?.sources?.total_count ?? 0; - const subs = this.props.customer?.subscriptions?.total_count ?? 0; - const invoices = this.props.invoices?.get("total_count") ?? 0; - return { cards, subs, invoices }; - } - - private render_page(): Rendered { - if (!this.props.for_course) return; - if (!this.props.loaded) { - // nothing loaded yet from backend - return ; - } else if (this.props.customer == null && this.props.for_course) { - // user not initialized yet -- only thing to do is add a card. - return ( -
    - -
    - ); - } else { - // data loaded and customer exists - if (this.props.customer == null) return; // can't happen; for typescript - const { subs } = this.counts(); - if (this.props.is_simplified && subs > 0) { - return ( -
    - -
    - ); - } else if (this.props.is_simplified) { - return ( -
    - -
    - ); - } - } - } - - renderLinks() { - return ( -
    -

    Links

    -
      - {!this.props.for_course ? this.render_help_suggestion() : undefined} - {!this.props.no_stripe ? this.render_action() : undefined} - {this.render_error()} - {!this.props.no_stripe ? this.render_page() : undefined} -
    -
    - ); - } - - public render(): Rendered { - return ( - <> - {this.renderLinks()} - {!this.props.is_simplified ?
    : undefined} - - ); - } - } -); diff --git a/src/packages/frontend/billing/confirm-payment-method.tsx b/src/packages/frontend/billing/confirm-payment-method.tsx deleted file mode 100644 index 989875eaf9..0000000000 --- a/src/packages/frontend/billing/confirm-payment-method.tsx +++ /dev/null @@ -1,88 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -import { Alert, Well } from "@cocalc/frontend/antd-bootstrap"; -import { Component, Rendered } from "../app-framework"; -import { Gap } from "../components/gap"; -import { Icon } from "../components/icon"; - -import { Customer, Source } from "./types"; -import { AddPaymentMethod } from "./add-payment-method"; -import { PaymentMethod } from "./payment-method"; - -interface Props { - customer?: Customer; - is_recurring: boolean; - on_close: Function; -} - -export class ConfirmPaymentMethod extends Component { - private render_single_payment_confirmation(): Rendered { - if (this.props.is_recurring) return; - return ( - -

    Payment will be processed with the card below.

    -

    To change payment methods, please change your default card above.

    -     - ); - } - - private render_recurring_payment_confirmation(): Rendered { - if (!this.props.is_recurring) return; - return ( - -

    - The initial payment will be processed with the card below. Future - payments will be made with whichever card you have set as your default - - at the time of renewal. -

    -     - ); - } - - private default_card(): Source | undefined { - if ( - this.props.customer == null || - this.props.customer.sources.data.length == 0 - ) { - // no card - return; - } - - for (const card_data of this.props.customer.sources.data) { - if (card_data.id === this.props.customer.default_source) { - return card_data; - } - } - // Should not happen (there should always be a default), but - // it did: https://github.com/sagemathinc/cocalc/issues/3468 - // We try again with whatever the first card is. - for (const card_data of this.props.customer.sources.data) { - return card_data; - } - // Still no card? This should also never happen since we - // checked the length above. Returns undefined which asks for card. - } - - public render(): Rendered { - const default_card: Source | undefined = this.default_card(); - if (default_card == null) { - return ; - } - return ( - -

    - Confirm your payment card -

    - {this.render_single_payment_confirmation()} - {this.render_recurring_payment_confirmation()} - - - -     - ); - } -} diff --git a/src/packages/frontend/billing/payment-method.tsx b/src/packages/frontend/billing/payment-method.tsx deleted file mode 100644 index 26e28ac894..0000000000 --- a/src/packages/frontend/billing/payment-method.tsx +++ /dev/null @@ -1,162 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -import { Rendered, useState } from "../app-framework"; -import { - Alert, - Button, - ButtonToolbar, - Row, - Col, -} from "@cocalc/frontend/antd-bootstrap"; -import { Icon, IconName } from "../components/icon"; -import { Gap } from "../components/gap"; -import { brand_to_icon_name } from "./data"; - -import { Source } from "./types"; - -interface Props { - source: Source; - default?: boolean; // required for set_as_default - set_as_default?: Function; // called when this card should be set to default - delete_method?: Function; // called when this card should be deleted -} - -export const PaymentMethod: React.FC = (props) => { - const [confirm_default, set_confirm_default] = useState(false); - const [confirm_delete, set_confirm_delete] = useState(false); - - function icon_name(): IconName { - return brand_to_icon_name( - props.source.brand != null ? props.source.brand.toLowerCase() : undefined, - ); - } - - function render_confirm_default(): Rendered { - return ( - - - -

    - Are you sure you want to set this payment card to be the default? -

    -

    - All future payments will be made with the card that is the default{" "} - at the time of renewal. Changing your default card right - before a subscription renewal will cause the - new default to be charged instead of the previous one. -

    - - - - - - - -     -     - ); - } - - function render_confirm_delete(): Rendered { - return ( - - - - Are you sure you want to delete this payment method? - - - - - - - - - - ); - } - - function render_card(): Rendered { - return ( - - - {props.source.brand} - - - ···· - {props.source.last4} - - - {props.source.exp_month}/{props.source.exp_year} - - {props.source.name} - {props.source.address_country} - - {props.source.address_state} - - - {props.source.address_zip} - - {props.set_as_default != null || props.delete_method != null - ? render_action_buttons() - : undefined} - - ); - } - - function render_action_buttons(): Rendered { - return ( - - - {props.set_as_default != null ? ( - - ) : undefined} - - {props.delete_method != null ? ( - - ) : undefined} - - - ); - } - - return ( -
    - {render_card()} - {confirm_default ? render_confirm_default() : undefined} - {confirm_delete ? render_confirm_delete() : undefined} -
    - ); -}; diff --git a/src/packages/frontend/billing/payment-methods.tsx b/src/packages/frontend/billing/payment-methods.tsx deleted file mode 100644 index eedf3a92b2..0000000000 --- a/src/packages/frontend/billing/payment-methods.tsx +++ /dev/null @@ -1,113 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -import { Rendered, useActions, useState } from "../app-framework"; -import { Button, Row, Col, Panel } from "../antd-bootstrap"; -import { Icon } from "../components/icon"; -import { Source } from "./types"; -import { AddPaymentMethod } from "./add-payment-method"; -import { PaymentMethod } from "./payment-method"; -import { ErrorDisplay } from "../components/error-display"; - -import { cmp } from "@cocalc/util/misc"; - -interface Props { - sources?: { data: Source[] }; // could be undefined, if it is a customer and all sources are removed - default?: string; -} - -type State = "view" | "delete" | "add_new"; - -export const PaymentMethods: React.FC = (props) => { - const [state, set_state] = useState("view"); - const [error, set_error] = useState(""); - const actions = useActions("billing"); - - function add_payment_method(): void { - set_state("add_new"); - } - - function render_add_payment_method(): Rendered { - if (state === "add_new") { - return set_state("view")} />; - } - } - - function render_add_payment_method_button(): Rendered { - return ( - - ); - } - - function render_header(): Rendered { - return ( - - - Payment methods - - {render_add_payment_method_button()} - - ); - } - - function set_as_default(id: string): void { - actions.set_as_default_payment_method(id); - } - - function delete_method(id: string): void { - actions.delete_payment_method(id); - } - - function render_payment_method(source: Source): Rendered { - if (source.object != "card") { - // TODO: non credit cards not yet supported. - // These *do* arise naturally already in cocalc, e.g., when you pay via - // for an invoice with a failing payment directly on the stripe page - // for your invoice. - return; - } - return ( - set_as_default(source.id)} - delete_method={() => delete_method(source.id)} - /> - ); - } - - function render_payment_methods(): undefined | Rendered[] { - // this happens, when it is a customer but all credit cards are deleted! - if (props.sources == null) { - return; - } - // Always sort sources in the same order. This way when you select - // a default source, they don't get reordered, which is really confusing. - props.sources.data.sort((a, b) => cmp(a.id, b.id)); - return props.sources.data.map((source) => render_payment_method(source)); - } - - function render_error(): Rendered { - if (error) { - return set_error("")} />; - } - } - - return ( - - {render_error()} - {state == "add_new" ? render_add_payment_method() : undefined} - {render_payment_methods()} - - ); -}; diff --git a/src/packages/frontend/chat/chat-indicator.tsx b/src/packages/frontend/chat/chat-indicator.tsx index e62e81db03..d1e60e6707 100644 --- a/src/packages/frontend/chat/chat-indicator.tsx +++ b/src/packages/frontend/chat/chat-indicator.tsx @@ -29,8 +29,10 @@ export type ChatState = const CHAT_INDICATOR_STYLE: React.CSSProperties = { fontSize: "15pt", - paddingTop: "3px", + paddingTop: "2px", cursor: "pointer", + background: "#e8e8e8", + borderTop: "2px solid lightgrey", } as const; const USERS_VIEWING_STYLE: React.CSSProperties = { @@ -111,7 +113,7 @@ function ChatButton({ project_id, path, chatState }) { danger={isNewChat} className={isNewChat ? "smc-chat-notification" : undefined} onClick={toggleChat} - style={{ color: chatState ? "orange" : "#333" }} + style={{ background: chatState ? "white" : undefined }} > diff --git a/src/packages/frontend/chat/video/video-chat.ts b/src/packages/frontend/chat/video/video-chat.ts index 515d2d8f60..6d4a5287cc 100644 --- a/src/packages/frontend/chat/video/video-chat.ts +++ b/src/packages/frontend/chat/video/video-chat.ts @@ -92,7 +92,7 @@ export class VideoChat { private chatroomId = (): string => { const secret_token = redux .getStore("projects") - .getIn(["project_map", this.project_id, "status", "secret_token"]); + .getIn(["project_map", this.project_id, "secret_token"]); if (!secret_token) { alert_message({ type: "error", diff --git a/src/packages/frontend/client/account.ts b/src/packages/frontend/client/account.ts index 5f8bc43b25..68b2db5efd 100644 --- a/src/packages/frontend/client/account.ts +++ b/src/packages/frontend/client/account.ts @@ -5,74 +5,18 @@ import { callback } from "awaiting"; declare const $: any; // jQuery -import * as message from "@cocalc/util/message"; -import { AsyncCall, WebappClient } from "./client"; +import { WebappClient } from "./client"; import type { ApiKey } from "@cocalc/util/db-schema/api-keys"; import api from "./api"; export class AccountClient { - private async_call: AsyncCall; private client: WebappClient; - private create_account_lock: boolean = false; constructor(client: WebappClient) { this.client = client; - this.async_call = client.async_call; } - private async call(message): Promise { - return await this.async_call({ - message, - allow_post: false, // never works or safe for account related functionality - timeout: 30, // 30s for all account stuff. - }); - } - - public async create_account(opts: { - first_name?: string; - last_name?: string; - email_address?: string; - password?: string; - agreed_to_terms?: boolean; - usage_intent?: string; - get_api_key?: boolean; // if given, will create/get api token in response message - token?: string; // only required if an admin set the account creation token. - }): Promise { - if (this.create_account_lock) { - // don't allow more than one create_account message at once -- see https://github.com/sagemathinc/cocalc/issues/1187 - return message.account_creation_failed({ - reason: { - account_creation_failed: - "You are submitting too many requests to create an account; please wait a second.", - }, - }); - } - - try { - this.create_account_lock = true; - return await this.call(message.create_account(opts)); - } finally { - setTimeout(() => (this.create_account_lock = false), 1500); - } - } - - public async delete_account(account_id: string): Promise { - return await this.call(message.delete_account({ account_id })); - } - - public async sign_in(opts: { - email_address: string; - password: string; - remember_me?: boolean; - get_api_key?: boolean; // if given, will create/get api token in response message - }): Promise { - return await this.async_call({ - message: message.sign_in(opts), - error_event: false, - }); - } - - public async cookies(mesg): Promise { + cookies = async (mesg): Promise => { const f = (cb) => { const j = $.ajax({ url: mesg.url, @@ -82,105 +26,58 @@ export class AccountClient { j.fail(() => cb("failed")); }; await callback(f); - } + }; - public async sign_out(everywhere: boolean = false): Promise { + sign_out = async (everywhere: boolean = false): Promise => { await api("/accounts/sign-out", { all: everywhere }); delete this.client.account_id; - await this.call(message.sign_out({ everywhere })); this.client.emit("signed_out"); - } + }; - public async change_password( + change_password = async ( currentPassword: string, newPassword: string = "", - ): Promise { + ): Promise => { await api("/accounts/set-password", { currentPassword, newPassword }); - } + }; - public async change_email( + change_email = async ( new_email_address: string, password: string = "", - ): Promise { + ): Promise => { if (this.client.account_id == null) { throw Error("must be logged in"); } - const x = await this.call( - message.change_email_address({ - account_id: this.client.account_id, - new_email_address, - password, - }), - ); - if (x.error) { - throw Error(x.error); - } - } + await api("accounts/set-email-address", { + email_address: new_email_address, + password, + }); + }; - public async send_verification_email( + send_verification_email = async ( only_verify: boolean = true, - ): Promise { - const account_id = this.client.account_id; - if (!account_id) { - throw Error("must be signed in to an account"); - } - const x = await this.call( - message.send_verification_email({ - account_id, - only_verify, - }), - ); - if (x.error) { - throw Error(x.error); - } - } - - // forgot password -- send forgot password request to server - public async forgot_password(email_address: string): Promise { - const x = await this.call( - message.forgot_password({ - email_address, - }), - ); - if (x.error) { - throw Error(x.error); - } - } - - // forgot password -- send forgot password request to server - public async reset_forgot_password( - reset_code: string, - new_password: string, - ): Promise { - const resp = await this.call( - message.reset_forgot_password({ - reset_code, - new_password, - }), - ); - if (resp.error) { - throw Error(resp.error); - } - } + ): Promise => { + await this.client.conat_client.hub.system.sendEmailVerification({ + only_verify, + }); + }; // forget about a given passport authentication strategy for this user - public async unlink_passport(strategy: string, id: string): Promise { - return await this.call( - message.unlink_passport({ - strategy, - id, - }), - ); - } + unlink_passport = async (strategy: string, id: string): Promise => { + await this.client.conat_client.hub.system.deletePassport({ + strategy, + id, + }); + }; // new interface: getting, setting, editing, deleting, etc., the api keys for a project - public async api_keys(opts: { + api_keys = async (opts: { action: "get" | "delete" | "create" | "edit"; password?: string; name?: string; id?: number; expire?: Date; - }): Promise { - return await this.client.nats_client.hub.system.manageApiKeys(opts); - } + }): Promise => { + return await this.client.conat_client.hub.system.manageApiKeys(opts); + }; } diff --git a/src/packages/frontend/client/admin.ts b/src/packages/frontend/client/admin.ts index 00a30d7315..b2e8ddaa93 100644 --- a/src/packages/frontend/client/admin.ts +++ b/src/packages/frontend/client/admin.ts @@ -3,7 +3,6 @@ * License: MS-RSL – see LICENSE.md for details */ -import * as message from "@cocalc/util/message"; import type { WebappClient } from "./client"; import api from "./api"; @@ -14,17 +13,6 @@ export class AdminClient { this.client = client; } - public async admin_reset_password(email_address: string): Promise { - return ( - await this.client.async_call({ - message: message.admin_reset_password({ - email_address, - }), - allow_post: true, - }) - ).link; - } - public async admin_ban_user( account_id: string, ban: boolean = true, // if true, ban user -- if false, remove ban @@ -37,7 +25,7 @@ export class AdminClient { } public async get_user_auth_token(user_account_id: string): Promise { - return await this.client.nats_client.hub.system.generateUserAuthToken({ + return await this.client.conat_client.hub.system.generateUserAuthToken({ user_account_id, }); } diff --git a/src/packages/frontend/client/anonymous-setup.ts b/src/packages/frontend/client/anonymous-setup.ts deleted file mode 100644 index 5d383ac68a..0000000000 --- a/src/packages/frontend/client/anonymous-setup.ts +++ /dev/null @@ -1,110 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -import { once } from "@cocalc/util/async-utils"; -import { redux } from "../app-framework"; -import { QueryParams } from "../misc/query-params"; -import { WelcomeFile } from "./welcome-file"; -import { WebappClient } from "./client"; -import { PROJECT_INVITE_QUERY_PARAM } from "../collaborators/handle-project-invite"; -import { hasRememberMe } from "@cocalc/frontend/misc/remember-me"; -import { appBasePath } from "@cocalc/frontend/customize/app-base-path"; - -export const ANON_PROJECT_TITLE = "Welcome to CoCalc!"; - -/* -should_do_anonymous_setup: Determine if the anonymous query param is set at all -(doesn't matter to what) during initial page load. Similar, if the -project_invite query param is set, this implies anonymous, so we also do anon -setup there if the user isn't already (likely) signed in. - -Also do NOT make true if has_remember_me is set, since then probably -the user has an account. -*/ -let project_invite_query_param = QueryParams.get(PROJECT_INVITE_QUERY_PARAM); -export function should_do_anonymous_setup(): boolean { - const anonymous_query_param = QueryParams.get("anonymous"); - return ( - (anonymous_query_param != null || project_invite_query_param != null) && - !hasRememberMe(appBasePath) - ); -} - -async function setup_default_project(log) { - const actions = redux.getActions("projects"); - log("creating project"); - const project_id = await actions.create_project({ - title: ANON_PROJECT_TITLE, - start: true, - description: "", - }); - log("opening project"); - actions.open_project({ project_id, switch_to: true }); - await new WelcomeFile(project_id).open(); -} - -export async function do_anonymous_setup(client: WebappClient): Promise { - function log(..._args): void { - // uncomment to debug... - // console.log("do_anonymous_setup", ..._args); - } - log(); - try { - redux.getActions("account").setState({ doing_anonymous_setup: true }); - log("creating account"); - try { - const resp = await client.account_client.create_account({ - first_name: "Anonymous", - last_name: `User-${Math.round(Date.now() / 1000)}`, - }); - if (resp?.event == "account_creation_failed") { - throw Error(resp.error); - } - } catch (err) { - log("failed to create account", err); - // If there is an error specifically with creating the account - // due to the backend not allowing it (e.g., missing token), then - // it is fine to silently return, which falls back to the login - // screen. Of course, all other errors below should make some noise. - return; - } - if (!client.is_signed_in()) { - log("waiting to be signed in"); - await once(client, "signed_in"); - } - if (project_invite_query_param) { - // This will get handled elsewhere. In particular, we - // don't need to do anything else besides make - // their anonymous account. - return; - } - - // "share" and "custom software images" create projects on their own! - const launch_store = redux.getStore( - (await import("../launch/actions")).NAME - ); - const need_project = !launch_store.get("type"); - if (need_project) { - await setup_default_project(log); - } - } catch (err) { - console.warn("ERROR doing anonymous sign up -- ", err); - log("err", err); - // There was an error creating the account (probably), so we do nothing - // further involving making an anonymous account. - // If the user didn't get signed in, this will fallback to sign in page, which - // is reasonable behavior. - // Such an error *should* happen if, e.g., a sign in token is required, - // or maybe this user's ip is blocked. Falling back - // to normal sign up makes sense in this case. - return; - } finally { - redux.getActions("account").setState({ doing_anonymous_setup: false }); - log("removing anonymous param"); - // In all cases, remove the 'anonymous' parameter. This way if - // they refresh their browser it won't cause confusion. - QueryParams.remove("anonymous"); - } -} diff --git a/src/packages/frontend/client/client.ts b/src/packages/frontend/client/client.ts index fbaba53a21..08dec77e57 100644 --- a/src/packages/frontend/client/client.ts +++ b/src/packages/frontend/client/client.ts @@ -6,7 +6,6 @@ import { bind_methods } from "@cocalc/util/misc"; import { EventEmitter } from "events"; import { delay } from "awaiting"; import { alert_message } from "../alerts"; -import { StripeClient } from "./stripe"; import { ProjectCollaborators } from "./project-collaborators"; import { Messages } from "./messages"; import { QueryClient } from "./query"; @@ -16,13 +15,11 @@ import { ProjectClient } from "./project"; import { AdminClient } from "./admin"; import { LLMClient } from "./llm"; import { PurchasesClient } from "./purchases"; -import { JupyterClient } from "./jupyter"; import { SyncClient } from "@cocalc/sync/client/sync-client"; import { UsersClient } from "./users"; import { FileClient } from "./file"; import { TrackingClient } from "./tracking"; -import { NatsClient } from "@cocalc/frontend/nats/client"; -import { HubClient } from "./hub"; +import { ConatClient } from "@cocalc/frontend/conat/client"; import { IdleClient } from "./idle"; import { version } from "@cocalc/util/smc-version"; import { setup_global_cocalc } from "./console"; @@ -32,13 +29,12 @@ import Cookies from "js-cookie"; import { basePathCookieName } from "@cocalc/util/misc"; import { ACCOUNT_ID_COOKIE_NAME } from "@cocalc/util/db-schema/accounts"; import { appBasePath } from "@cocalc/frontend/customize/app-base-path"; -import type { NatsSyncTableFunction } from "@cocalc/nats/sync/synctable"; +import type { ConatSyncTableFunction } from "@cocalc/conat/sync/synctable"; import type { - CallNatsServiceFunction, - CreateNatsServiceFunction, -} from "@cocalc/nats/service"; -import type { NatsEnvFunction } from "@cocalc/nats/types"; -import { randomId } from "@cocalc/nats/names"; + CallConatServiceFunction, + CreateConatServiceFunction, +} from "@cocalc/conat/service"; +import { randomId } from "@cocalc/conat/names"; // This DEBUG variable comes from webpack: declare const DEBUG; @@ -64,7 +60,6 @@ export type AsyncCall = (opts: object) => Promise; export interface WebappClient extends EventEmitter { account_id?: string; browser_id: string; - stripe: StripeClient; project_collaborators: ProjectCollaborators; messages: Messages; query_client: QueryClient; @@ -74,13 +69,11 @@ export interface WebappClient extends EventEmitter { admin_client: AdminClient; openai_client: LLMClient; purchases_client: PurchasesClient; - jupyter_client: JupyterClient; sync_client: SyncClient; users_client: UsersClient; file_client: FileClient; tracking_client: TrackingClient; - nats_client: NatsClient; - hub_client: HubClient; + conat_client: ConatClient; idle_client: IdleClient; client: Client; @@ -91,11 +84,10 @@ export interface WebappClient extends EventEmitter { get_username: Function; is_signed_in: () => boolean; synctable_project: Function; - synctable_nats: NatsSyncTableFunction; - callNatsService: CallNatsServiceFunction; - createNatsService: CreateNatsServiceFunction; - getNatsEnv: NatsEnvFunction; - pubsub_nats: Function; + synctable_conat: ConatSyncTableFunction; + callConatService: CallConatServiceFunction; + createConatService: CreateConatServiceFunction; + pubsub_conat: Function; prettier: Function; exec: Function; touch_project: (project_id: string, compute_server_id?: number) => void; @@ -106,7 +98,6 @@ export interface WebappClient extends EventEmitter { buffer_path: string, ) => Promise; log_error: (any) => void; - async_call: AsyncCall; user_tracking: Function; send: Function; call: Function; @@ -122,6 +113,7 @@ export interface WebappClient extends EventEmitter { mark_file: (opts: any) => Promise; set_connected?: Function; version: Function; + alert_message: Function; } export const WebappClient = null; // webpack + TS es2020 modules need this @@ -148,7 +140,6 @@ Connection events: class Client extends EventEmitter implements WebappClient { account_id: string = Cookies.get(ACCOUNT_ID_COOKIE); browser_id: string = randomId(); - stripe: StripeClient; project_collaborators: ProjectCollaborators; messages: Messages; query_client: QueryClient; @@ -158,13 +149,11 @@ class Client extends EventEmitter implements WebappClient { admin_client: AdminClient; openai_client: LLMClient; purchases_client: PurchasesClient; - jupyter_client: JupyterClient; sync_client: SyncClient; users_client: UsersClient; file_client: FileClient; tracking_client: TrackingClient; - nats_client: NatsClient; - hub_client: HubClient; + conat_client: ConatClient; idle_client: IdleClient; client: Client; @@ -175,11 +164,10 @@ class Client extends EventEmitter implements WebappClient { get_username: Function; is_signed_in: () => boolean; synctable_project: Function; - synctable_nats: NatsSyncTableFunction; - callNatsService: CallNatsServiceFunction; - createNatsService: CreateNatsServiceFunction; - getNatsEnv: NatsEnvFunction; - pubsub_nats: Function; + synctable_conat: ConatSyncTableFunction; + callConatService: CallConatServiceFunction; + createConatService: CreateConatServiceFunction; + pubsub_conat: Function; prettier: Function; exec: Function; touch_project: (project_id: string, compute_server_id?: number) => void; @@ -191,7 +179,6 @@ class Client extends EventEmitter implements WebappClient { ) => Promise; log_error: (any) => void; - async_call: AsyncCall; user_tracking: Function; send: Function; call: Function; @@ -217,17 +204,6 @@ class Client extends EventEmitter implements WebappClient { return (..._) => {}; }; } - this.hub_client = bind_methods(new HubClient(this)); - this.is_signed_in = this.hub_client.is_signed_in.bind(this.hub_client); - this.is_connected = this.hub_client.is_connected.bind(this.hub_client); - this.call = this.hub_client.call.bind(this.hub_client); - this.async_call = this.hub_client.async_call.bind(this.hub_client); - this.latency = this.hub_client.latency.bind(this.hub_client); - - this.stripe = bind_methods(new StripeClient(this.call.bind(this))); - this.project_collaborators = bind_methods( - new ProjectCollaborators(this.async_call.bind(this)), - ); this.messages = new Messages(); this.query_client = bind_methods(new QueryClient(this)); this.time_client = bind_methods(new TimeClient(this)); @@ -241,14 +217,14 @@ class Client extends EventEmitter implements WebappClient { this.admin_client = bind_methods(new AdminClient(this)); this.openai_client = bind_methods(new LLMClient(this)); this.purchases_client = bind_methods(new PurchasesClient(this)); - this.jupyter_client = bind_methods( - new JupyterClient(this.async_call.bind(this)), - ); this.users_client = bind_methods(new UsersClient(this)); this.tracking_client = bind_methods(new TrackingClient(this)); - this.nats_client = bind_methods(new NatsClient(this)); - this.file_client = bind_methods(new FileClient(this.async_call.bind(this))); + this.conat_client = bind_methods(new ConatClient(this)); + this.is_signed_in = this.conat_client.is_signed_in.bind(this.conat_client); + this.is_connected = this.conat_client.is_connected.bind(this.conat_client); + this.file_client = bind_methods(new FileClient()); this.idle_client = bind_methods(new IdleClient(this)); + this.project_collaborators = bind_methods(new ProjectCollaborators(this)); // must be after this.conat_client is defined. // Expose a public API as promised by WebappClient this.server_time = this.time_client.server_time.bind(this.time_client); @@ -266,14 +242,10 @@ class Client extends EventEmitter implements WebappClient { this.synctable_database = this.sync_client.synctable_database.bind( this.sync_client, ); - this.synctable_project = this.sync_client.synctable_project.bind( - this.sync_client, - ); - this.synctable_nats = this.nats_client.synctable; - this.pubsub_nats = this.nats_client.pubsub; - this.callNatsService = this.nats_client.callNatsService; - this.createNatsService = this.nats_client.createNatsService; - this.getNatsEnv = this.nats_client.getEnv; + this.synctable_conat = this.conat_client.synctable; + this.pubsub_conat = this.conat_client.pubsub; + this.callConatService = this.conat_client.callConatService; + this.createConatService = this.conat_client.createConatService; this.query = this.query_client.query.bind(this.query_client); this.async_query = this.query_client.query.bind(this.query_client); @@ -350,19 +322,20 @@ class Client extends EventEmitter implements WebappClient { project_id, path, setNotDeleted, - // id + doctype, }: { project_id: string; path: string; id?: number; + doctype?; // if file is deleted, this explicitly undeletes it. setNotDeleted?: boolean; }) => { - const x = await this.nats_client.openFiles(project_id); + const x = await this.conat_client.openFiles(project_id); if (setNotDeleted) { x.setNotDeleted(path); } - x.touch(path); + x.touch(path, doctype); }; } diff --git a/src/packages/frontend/client/console.ts b/src/packages/frontend/client/console.ts index 925b839e40..f0f686a2f7 100644 --- a/src/packages/frontend/client/console.ts +++ b/src/packages/frontend/client/console.ts @@ -45,7 +45,7 @@ export function setup_global_cocalc(client): void { const cocalc: any = window.cc ?? {}; cocalc.client = client; - cocalc.nats = client.nats_client; + cocalc.conat = client.conat_client; cocalc.misc = require("@cocalc/util/misc"); cocalc.immutable = require("immutable"); cocalc.done = cocalc.misc.done; diff --git a/src/packages/frontend/client/file.ts b/src/packages/frontend/client/file.ts index 8000a8c999..7103e3da64 100644 --- a/src/packages/frontend/client/file.ts +++ b/src/packages/frontend/client/file.ts @@ -3,33 +3,11 @@ * License: MS-RSL – see LICENSE.md for details */ -import * as message from "@cocalc/util/message"; -import { AsyncCall } from "./client"; import { redux } from "../app-framework"; import { required, defaults } from "@cocalc/util/misc"; export class FileClient { - private async_call: AsyncCall; - - constructor(async_call: AsyncCall) { - this.async_call = async_call; - } - - // Currently only used for testing and development in the console. - public async syncdoc_history( - string_id: string, - patches?: boolean, - ): Promise { - return ( - await this.async_call({ - message: message.get_syncdoc_history({ - string_id, - patches, - }), - allow_post: false, - }) - ).history; - } + constructor() {} // Returns true if the given file in the given project is currently // marked as deleted. @@ -61,13 +39,4 @@ export class FileClient { .getActions("file_use") ?.mark_file(opts.project_id, opts.path, opts.action, opts.ttl); } - - public async remove_blob_ttls( - uuids: string[], // list of sha1 hashes of blobs stored in the blobstore - ) { - if (uuids.length === 0) return; - await this.async_call({ - message: message.remove_blob_ttls({ uuids }), - }); - } } diff --git a/src/packages/frontend/client/hub.ts b/src/packages/frontend/client/hub.ts deleted file mode 100644 index b9a2433e6c..0000000000 --- a/src/packages/frontend/client/hub.ts +++ /dev/null @@ -1,525 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -import { callback, delay } from "awaiting"; -import { throttle } from "lodash"; -import type { WebappClient } from "./client"; -import { delete_cookie } from "../misc/cookies"; -import { - copy_without, - from_json_socket, - to_json_socket, - defaults, - required, - uuid, -} from "@cocalc/util/misc"; -import * as message from "@cocalc/util/message"; -import { - do_anonymous_setup, - should_do_anonymous_setup, -} from "./anonymous-setup"; -import { - deleteRememberMe, - setRememberMe, -} from "@cocalc/frontend/misc/remember-me"; -import { appBasePath } from "@cocalc/frontend/customize/app-base-path"; - -// Maximum number of outstanding concurrent messages (that have responses) -// to send at once to hub-websocket. -const MAX_CONCURRENT: number = 17; - -// just define what we need for code sanity -interface PrimusConnection { - write: (arg0: string) => void; - open: () => void; - end: () => void; - latency?: number; -} - -export interface MessageInfo { - count: number; - sent: number; - sent_length: number; - recv: number; - recv_length: number; - enqueued: number; - max_concurrent: number; -} - -export class HubClient { - private client: WebappClient; - private conn?: PrimusConnection; - - private connected: boolean = false; - private connection_is_totally_dead: boolean = false; - private num_attempts: number = 0; - private signed_in: boolean = false; - private signed_in_time: number = 0; - private signed_in_mesg: object; - - private call_callbacks: { - [id: string]: { - timeout?: any; - error_event: boolean; - first: boolean; - cb: Function; - }; - } = {}; - - private mesg_data: { - queue: any[]; - count: number; - sent: number; - sent_length: number; - recv: number; - recv_length: number; - } = { - queue: [], // messages in the queue to send - count: 0, // number of message currently outstanding - sent: 0, // total number of messages sent to backend. - sent_length: 0, // total amount of data sent - recv: 0, // number of messages received from backend - recv_length: 0, - }; - - constructor(client: WebappClient) { - this.client = client; - - /* We heavily throttle this, since it's ONLY used for the connections - dialog, which users never look at, and it could waste cpu trying to - update things for no reason. It also impacts the color of the - connection indicator, so throttling will make that color change a - bit more laggy. That's probably worth it. */ - this.emit_mesg_data = throttle(this.emit_mesg_data.bind(this), 2000); - - // never attempt to reconnect more than once per 10s, no matter what. - this.reconnect = throttle(this.reconnect.bind(this), 10000); - - // Start attempting to connect to a hub. - this.init_hub_websocket(); - } - - private emit_mesg_data(): void { - const info: MessageInfo = copy_without(this.mesg_data, ["queue"]) as any; - info.enqueued = this.mesg_data.queue.length; - info.max_concurrent = MAX_CONCURRENT; - this.client.emit("mesg_info", info); - } - - public get_num_attempts(): number { - return this.num_attempts; - } - - public send(mesg: object): void { - // uncomment this to work on removing the hub websocket connection entirely. - // console.log("send to hub", mesg); - const data = to_json_socket(mesg); - this.mesg_data.sent_length += data.length; - this.emit_mesg_data(); - this.write_data(data); - } - - private write_data(data: string): void { - if (this.conn == null) { - console.warn( - "HubClient.write_data: can't write data since not connected", - ); - return; - } - try { - this.conn.write(data); - } catch (err) { - console.warn("HubClient.write_data", err); - } - } - - private delete_websocket_cookie(): void { - delete_cookie("SMCSERVERID3"); - } - - public is_signed_in(): boolean { - return this.is_connected() && !!this.signed_in; - } - - public set_signed_in(): void { - this.signed_in = true; - } - - public set_signed_out(): void { - this.signed_in = false; - } - - public get_signed_in_time(): number { - return this.signed_in_time; - } - - public get_signed_in_mesg(): object { - return this.signed_in_mesg; - } - - public is_connected(): boolean { - return !!this.connected; - } - - public reconnect(): void { - if (this.connection_is_totally_dead) { - // CRITICAL: See https://github.com/primus/primus#primusopen ! - this.conn?.open(); - } - } - - public disconnect(): void { - if (this.connected) { - this.conn?.end(); - } - } - - private ondata(data: string): void { - //console.log("got #{data.length} of data") - this.mesg_data.recv += 1; - this.mesg_data.recv_length += data.length; - this.emit_mesg_data(); - this.handle_json_data(data); - } - - private async handle_json_data(data: string): Promise { - this.emit_mesg_data(); - const mesg = from_json_socket(data); - // console.log(`handle_json_data: ${data}`); - switch (mesg.event) { - case "cookies": - try { - await this.client.account_client.cookies(mesg); - } catch (err) { - console.warn("Error handling cookie ", mesg, err); - } - break; - - case "signed_in": - this.client.account_id = mesg.account_id; - this.set_signed_in(); - this.signed_in_time = Date.now(); - setRememberMe(appBasePath); - this.signed_in_mesg = mesg; - this.client.emit("signed_in", mesg); - break; - - case "remember_me_failed": - deleteRememberMe(appBasePath); - this.client.emit(mesg.event, mesg); - break; - - case "version": - this.client.emit("new_version", { - version: mesg.version, - min_version: mesg.min_version, - }); - break; - - case "error": - // An error that isn't tagged with an id -- some sort of general problem. - if (mesg.id == null) { - console.log(`WARNING: ${JSON.stringify(mesg.error)}`); - return; - } - break; - } - - // the call f(null, mesg) below can mutate mesg (!), so we better save the id here. - const { id } = mesg; - const v = this.call_callbacks[id]; - if (v != null) { - const { cb, error_event } = v; - v.first = false; - if (error_event && mesg.event === "error") { - if (!mesg.error) { - // make sure mesg.error is set to something. - mesg.error = "error"; - } - cb(mesg.error); - } else { - cb(undefined, mesg); - } - if (!mesg.multi_response) { - delete this.call_callbacks[id]; - } - } - } - - private do_call(opts: any, cb: Function): void { - if (opts.cb == null) { - // console.log("no opts.cb", opts.message) - // A call to the backend, but where we do not wait for a response. - // In order to maintain at least roughly our limit on MAX_CONCURRENT, - // we simply pretend that this message takes about 150ms - // to complete. This helps space things out so the server can - // handle requests properly, instead of just discarding them (be nice - // to the backend and it will be nice to you). - this.send(opts.message); - setTimeout(cb, 150); - return; - } - if (opts.message.id == null) { - // Assign a uuid (usually we do this) - opts.message.id = uuid(); - } - const { id } = opts.message; - let called_cb: boolean = false; - if (this.call_callbacks[id] != null) { - // User is requesting to send a message with the same id as - // a currently outstanding message. This typically happens - // when disconnecting and reconnecting. It's critical to - // clear up the existing call before overwritting - // call_callbacks[id]. The point is the message id's are - // NOT at all guaranteed to be random. - this.clear_call(id); - } - - this.call_callbacks[id] = { - cb: (...args) => { - if (!called_cb) { - called_cb = true; - cb(); - } - // NOTE: opts.cb is always defined since otherwise - // we would have exited above. - if (opts.cb != null) { - opts.cb(...args); - } - }, - error_event: !!opts.error_event, - first: true, - }; - - this.send(opts.message); - - if (opts.timeout) { - this.call_callbacks[id].timeout = setTimeout(() => { - if (this.call_callbacks[id] == null || this.call_callbacks[id].first) { - const error = "Timeout after " + opts.timeout + " seconds"; - if (!called_cb) { - called_cb = true; - cb(); - } - if (opts.cb != null) { - opts.cb(error, message.error({ id, error })); - } - delete this.call_callbacks[id]; - } - }, opts.timeout * 1000); - } else { - // IMPORTANT: No matter what, we call cb within 60s; if we don't do this then - // in case opts.timeout isn't set but opts.cb is, but user disconnects, - // then cb would never get called, which throws off our call counter. - // Note that the input to cb doesn't matter. - const f = () => { - if (!called_cb) { - called_cb = true; - cb(); - } - }; - this.call_callbacks[id].timeout = setTimeout(f, 60 * 1000); - } - } - - public call(opts: any): void { - // This function: - // * Modifies the message by adding an id attribute with a random uuid value - // * Sends the message to the hub - // * When message comes back with that id, call the callback and delete it (if cb opts.cb is defined) - // The message will not be seen by @handle_message. - // * If the timeout is reached before any messages come back, delete the callback and stop listening. - // However, if the message later arrives it may still be handled by @handle_message. - opts = defaults(opts, { - message: required, - timeout: undefined, - error_event: false, // if true, turn error events into just a normal err - allow_post: undefined, // TODO: deprecated -- completely ignored and not used in any way. - cb: undefined, - }); - if (!this.is_connected()) { - if (opts.cb != null) { - opts.cb("not connected"); - } - return; - } - this.mesg_data.queue.push(opts); - this.mesg_data.sent += 1; - this.update_calls(); - } - - // like call above, but async and error_event defaults to TRUE, - // so an exception is raised on resp messages that have event='error'. - - public async async_call(opts: any): Promise { - const f = (cb) => { - opts.cb = cb; - this.call(opts); - }; - if (opts.error_event == null) { - opts.error_event = true; - } - return await callback(f); - } - - private update_calls(): void { - while ( - this.mesg_data.queue.length > 0 && - this.mesg_data.count < MAX_CONCURRENT - ) { - this.process_next_call(); - } - } - - private process_next_call(): void { - if (this.mesg_data.queue.length === 0) { - return; - } - this.mesg_data.count += 1; - const opts = this.mesg_data.queue.shift(); - this.emit_mesg_data(); - this.do_call(opts, () => { - this.mesg_data.count -= 1; - this.emit_mesg_data(); - this.update_calls(); - }); - } - - private clear_call(id: string): void { - const obj = this.call_callbacks[id]; - if (obj == null) return; - delete this.call_callbacks[id]; - obj.cb("disconnect"); - if (obj.timeout) { - clearTimeout(obj.timeout); - delete obj.timeout; - } - } - - private clear_call_queue(): void { - for (const id in this.call_callbacks) { - this.clear_call(id); - } - this.emit_mesg_data(); - } - - private async init_hub_websocket(): Promise { - const log = (...mesg) => console.log("hub_websocket -", ...mesg); - log("connect"); - this.client.emit("connecting"); - - this.client.on("connected", () => { - this.send_version(); - // Any outstanding calls made before connecting happened - // can't possibly succeed, so we clear all outstanding messages. - this.clear_call_queue(); - }); - - this.delete_websocket_cookie(); - // Important: window.Primus is usually defined when we get to the point - // of running this code. However, sometimes it doesn't -- timing is random - // and whether it is defined here depends on a hub being available to - // serve it up. So we just keep trying until it is defined. - // There is no need to back off or delay, since we aren't - // actually doing anything at all here in terms of work! - log("Loading global websocket client library from hub-websocket..."); - while (window.Primus == null) { - await delay(200); - } - log( - "Loaded global websocket library! Now creating websocket connection to hub-websocket...", - ); - const conn = (this.conn = new window.Primus({ - reconnect: { - max: 30000, - min: 3000, - retries: 1000, - }, - })); - - conn.on("open", async () => { - this.connected = true; - this.connection_is_totally_dead = false; - this.client.emit("connected"); - log("connected"); - this.num_attempts = 0; - - conn.removeAllListeners("data"); - conn.on("data", this.ondata.bind(this)); - - if (should_do_anonymous_setup()) { - do_anonymous_setup(this.client); - } - }); - - conn.on("outgoing::open", () => { - log("connecting"); - this.client.emit("connecting"); - }); - - conn.on("offline", () => { - log("offline"); - this.connected = this.signed_in = false; - this.client.emit("disconnected", "offline"); - }); - - conn.on("online", () => { - log("online"); - }); - - conn.on("message", (evt) => { - this.ondata(evt.data); - }); - - conn.on("error", (err) => { - log("error: ", err); - }); - // NOTE: we do NOT emit an error event in this case! See - // https://github.com/sagemathinc/cocalc/issues/1819 - // for extensive discussion. - - conn.on("close", () => { - log("closed"); - this.connected = this.signed_in = false; - this.client.emit("disconnected", "close"); - }); - - conn.on("end", () => { - this.connection_is_totally_dead = true; - }); - - conn.on("reconnect scheduled", (opts) => { - this.num_attempts = opts.attempt; - // This just informs everybody that we *are* disconnected. - this.client.emit("disconnected", "close"); - conn.removeAllListeners("data"); - this.delete_websocket_cookie(); - log( - `reconnect scheduled (attempt ${opts.attempt} out of ${opts.retries})`, - ); - }); - - conn.on("reconnect", () => { - this.client.emit("connecting"); - }); - } - - private send_version(): void { - this.send(message.version({ version: this.client.version() })); - } - - public fix_connection(): void { - this.delete_websocket_cookie(); - this.conn?.end(); - this.conn?.open(); - this.client.nats_client.reconnect(); - } - - public latency(): number | void { - if (this.connected) { - return this.conn?.latency; - } - } -} diff --git a/src/packages/frontend/client/idle.ts b/src/packages/frontend/client/idle.ts index 8569d7b917..ef566b4723 100644 --- a/src/packages/frontend/client/idle.ts +++ b/src/packages/frontend/client/idle.ts @@ -113,8 +113,7 @@ export class IdleClient { console.log("Entering standby mode"); this.standbyMode = true; // console.log("idle timeout: disconnect!"); - this.client.nats_client.standby(); - this.client.hub_client.disconnect(); + this.client.conat_client.standby(); disconnect_from_all_projects(); }, CHECK_INTERVAL / 2); } @@ -133,8 +132,7 @@ export class IdleClient { if (this.standbyMode) { this.standbyMode = false; console.log("Leaving standby mode"); - this.client.nats_client.resume(); - this.client.hub_client.reconnect(); + this.client.conat_client.resume(); } }; diff --git a/src/packages/frontend/client/jupyter.ts b/src/packages/frontend/client/jupyter.ts deleted file mode 100644 index 4a4b0a892f..0000000000 --- a/src/packages/frontend/client/jupyter.ts +++ /dev/null @@ -1,60 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -import * as message from "@cocalc/util/message"; -import { AsyncCall } from "./client"; -import type { KernelSpec } from "@cocalc/jupyter/types"; - -export class JupyterClient { - private async_call: AsyncCall; - - constructor(async_call: AsyncCall) { - this.async_call = async_call; - } - - public async kernels(project_id?: string): Promise { - const resp = await this.async_call({ - message: message.jupyter_kernels({ project_id }), - }); - if (resp.error) { - throw Error(resp.error); - } - return resp.kernels; - } - - public async execute({ - input, - kernel, - history, - hash, - tag = "", - project_id, - path, - }: { - input?: string; - kernel?: string; - history?: string[]; - hash?: string; - tag?: string; - project_id?: string; - path?: string; - }): Promise<{ output: object[]; time: Date; total_time_s: number } | null> { - const resp = await this.async_call({ - message: message.jupyter_execute({ - hash, - input, - kernel, - history, - tag, - project_id, - path, - }), - }); - if (resp.error) { - throw Error(resp.error); - } - return resp; - } -} diff --git a/src/packages/frontend/client/llm.ts b/src/packages/frontend/client/llm.ts index 821e24c841..93a0d129a6 100644 --- a/src/packages/frontend/client/llm.ts +++ b/src/packages/frontend/client/llm.ts @@ -5,15 +5,7 @@ import { delay } from "awaiting"; import { EventEmitter } from "events"; - import { redux } from "@cocalc/frontend/app-framework"; -import type { EmbeddingData } from "@cocalc/util/db-schema/llm"; -import { - MAX_EMBEDDINGS_TOKENS, - MAX_REMOVE_LIMIT, - MAX_SAVE_LIMIT, - MAX_SEARCH_LIMIT, -} from "@cocalc/util/db-schema/llm"; import { LanguageModel, LanguageServiceCore, @@ -21,7 +13,6 @@ import { isFreeModel, model2service, } from "@cocalc/util/db-schema/llm-utils"; -import * as message from "@cocalc/util/message"; import type { WebappClient } from "./client"; import type { History } from "./types"; import { @@ -43,15 +34,6 @@ interface QueryLLMProps { startStreamExplicitly?: boolean; } -interface EmbeddingsQuery { - scope: string | string[]; - limit: number; // client automatically deals with large limit by making multiple requests (i.e., there is no limit on the limit) - text?: string; - filter?: object; - selector?: { include?: string[]; exclude?: string[] }; - offset?: number | string; -} - export class LLMClient { private client: WebappClient; @@ -183,13 +165,13 @@ export class LLMClient { if (chatStream == null) { // not streaming - return await this.client.nats_client.llm(options); + return await this.client.conat_client.llm(options); } chatStream.once("start", async () => { // streaming version try { - await this.client.nats_client.llm({ + await this.client.conat_client.llm({ ...options, stream: chatStream.process, }); @@ -200,139 +182,6 @@ export class LLMClient { return "see stream for output"; } - - public async embeddings_search( - query: EmbeddingsQuery, - ): Promise<{ id: string; payload: object }[]> { - let limit = Math.min(MAX_SEARCH_LIMIT, query.limit); - const result = await this.embeddings_search_call({ ...query, limit }); - - if (result.length >= MAX_SEARCH_LIMIT) { - // get additional pages - while (true) { - const offset = - query.text == null ? result[result.length - 1].id : result.length; - const page = await this.embeddings_search_call({ - ...query, - limit, - offset, - }); - // Include the new elements - result.push(...page); - if (page.length < MAX_SEARCH_LIMIT) { - // didn't reach the limit, so we're done. - break; - } - } - } - return result; - } - - private async embeddings_search_call({ - scope, - limit, - text, - filter, - selector, - offset, - }: EmbeddingsQuery) { - text = text?.trim(); - const resp = await this.client.async_call({ - message: message.openai_embeddings_search({ - scope, - text, - filter, - limit, - selector, - offset, - }), - }); - return resp.matches; - } - - public async embeddings_save({ - project_id, - path, - data: data0, - }: { - project_id: string; - path: string; - data: EmbeddingData[]; - }): Promise { - this.assertHasNeuralSearch(); - const { truncateMessage } = await import("@cocalc/frontend/misc/llm"); - - // Make data be data0, but without mutate data0 - // and with any text truncated to fit in the - // embeddings limit. - const data: EmbeddingData[] = []; - for (const x of data0) { - const { text } = x; - if (typeof text != "string") { - throw Error("text must be a string"); - } - const text1 = truncateMessage(text, MAX_EMBEDDINGS_TOKENS); - if (text1.length != text.length) { - data.push({ ...x, text: text1 }); - } else { - data.push(x); - } - } - - const ids: string[] = []; - let v = data; - while (v.length > 0) { - const resp = await this.client.async_call({ - message: message.openai_embeddings_save({ - project_id, - path, - data: v.slice(0, MAX_SAVE_LIMIT), - }), - }); - ids.push(...resp.ids); - v = v.slice(MAX_SAVE_LIMIT); - } - - return ids; - } - - public async embeddings_remove({ - project_id, - path, - data, - }: { - project_id: string; - path: string; - data: EmbeddingData[]; - }): Promise { - this.assertHasNeuralSearch(); - - const ids: string[] = []; - let v = data; - while (v.length > 0) { - const resp = await this.client.async_call({ - message: message.openai_embeddings_remove({ - project_id, - path, - data: v.slice(0, MAX_REMOVE_LIMIT), - }), - }); - ids.push(...resp.ids); - v = v.slice(MAX_REMOVE_LIMIT); - } - - return ids; - } - - neuralSearchIsEnabled(): boolean { - return !!redux.getStore("customize").get("neural_search_enabled"); - } - - assertHasNeuralSearch() { - if (!this.neuralSearchIsEnabled()) { - throw Error("OpenAI support is not currently enabled on this server"); - } - } } class ChatStream extends EventEmitter { @@ -340,7 +189,7 @@ class ChatStream extends EventEmitter { super(); } - process = (text: string|null) => { + process = (text: string | null) => { // emits undefined text when done (or err below) this.emit("token", text); }; diff --git a/src/packages/frontend/client/project-collaborators.ts b/src/packages/frontend/client/project-collaborators.ts index 4f3ae3b1d5..2bec51fc51 100644 --- a/src/packages/frontend/client/project-collaborators.ts +++ b/src/packages/frontend/client/project-collaborators.ts @@ -3,18 +3,14 @@ * License: MS-RSL – see LICENSE.md for details */ -import * as message from "@cocalc/util/message"; -import { AsyncCall } from "./client"; +import type { ConatClient } from "@cocalc/frontend/conat/client"; +import type { AddCollaborator } from "@cocalc/conat/hub/api/projects"; export class ProjectCollaborators { - private async_call: AsyncCall; + private conat: ConatClient; - constructor(async_call: AsyncCall) { - this.async_call = async_call; - } - - private async call(message: object): Promise { - return await this.async_call({ message }); + constructor(client) { + this.conat = client.conat_client; } public async invite_noncloud(opts: { @@ -27,7 +23,9 @@ export class ProjectCollaborators { email: string; // body in HTML format subject?: string; }): Promise { - return await this.call(message.invite_noncloud_collaborators(opts)); + return await this.conat.hub.projects.inviteCollaboratorWithoutAccount({ + opts, + }); } public async invite(opts: { @@ -40,32 +38,28 @@ export class ProjectCollaborators { email?: string; subject?: string; }): Promise { - return await this.call(message.invite_collaborator(opts)); + return await this.conat.hub.projects.inviteCollaborator({ + opts, + }); } public async remove(opts: { project_id: string; account_id: string; }): Promise { - return await this.call(message.remove_collaborator(opts)); + return await this.conat.hub.projects.removeCollaborator({ + opts, + }); } // Directly add one (or more) collaborators to (one or more) projects via - // a single API call. There is no invite process, etc. + // a single API call. There is no defined invite email message. public async add_collaborator( - opts: - | { - project_id: string; - account_id: string; - } - | { - token_id: string; - account_id: string; - } - | { project_id: string[]; account_id: string[] } // for adding more than one at once - | { account_id: string[]; token_id: string[] } // for adding more than one at once - ): Promise<{ event: "error" | "ok"; project_id?: string | string[] }> { + opts: AddCollaborator, + ): Promise<{ project_id?: string | string[] }> { // project_id is a single string or an array of project id's in case of a token. - return await this.call(message.add_collaborator(opts)); + return await this.conat.hub.projects.addCollaborator({ + opts, + }); } } diff --git a/src/packages/frontend/client/project.ts b/src/packages/frontend/client/project.ts index f0644350cf..79ed25713f 100644 --- a/src/packages/frontend/client/project.ts +++ b/src/packages/frontend/client/project.ts @@ -16,18 +16,6 @@ import { allow_project_to_run } from "@cocalc/frontend/project/client-side-throt import { ensure_project_running } from "@cocalc/frontend/project/project-start-warning"; import { API } from "@cocalc/frontend/project/websocket/api"; import { connection_to_project } from "@cocalc/frontend/project/websocket/connect"; -import { - ProjectInfo, - project_info, -} from "@cocalc/frontend/project/websocket/project-info"; -import { - ProjectStatus, - project_status, -} from "@cocalc/frontend/project/websocket/project-status"; -import { - UsageInfoWS, - get_usage_info, -} from "@cocalc/frontend/project/websocket/usage-info"; import { Configuration, ConfigurationAspect, @@ -39,7 +27,6 @@ import { type ExecOpts, type ExecOutput, } from "@cocalc/util/db-schema/projects"; -import * as message from "@cocalc/util/message"; import { coerce_codomain_to_numbers, copy_without, @@ -50,11 +37,10 @@ import { } from "@cocalc/util/misc"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { DirectoryListingEntry } from "@cocalc/util/types"; -import httpApi from "./api"; import { WebappClient } from "./client"; import { throttle } from "lodash"; -import { writeFile, type WriteFileOptions } from "@cocalc/nats/files/write"; -import { readFile, type ReadFileOptions } from "@cocalc/nats/files/read"; +import { writeFile, type WriteFileOptions } from "@cocalc/conat/files/write"; +import { readFile, type ReadFileOptions } from "@cocalc/conat/files/read"; export class ProjectClient { private client: WebappClient; @@ -64,18 +50,14 @@ export class ProjectClient { this.client = client; } - private async call(message: object): Promise { - return await this.client.async_call({ message }); - } - - private natsApi = (project_id: string) => { - return this.client.nats_client.projectApi({ project_id }); + private conatApi = (project_id: string) => { + return this.client.conat_client.projectApi({ project_id }); }; // This can write small text files in one message. - public async write_text_file(opts): Promise { + write_text_file = async (opts): Promise => { await this.writeFile(opts); - } + }; // writeFile -- easily write **arbitrarily large text or binary files** // to a project from a readable stream or a string! @@ -101,26 +83,26 @@ export class ProjectClient { return Buffer.concat(chunks); }; - public async read_text_file({ + read_text_file = async ({ project_id, path, }: { project_id: string; // string or array of strings path: string; // string or array of strings - }): Promise { - return await this.natsApi(project_id).system.readTextFileFromProject({ + }): Promise => { + return await this.conatApi(project_id).system.readTextFileFromProject({ path, }); - } + }; // Like "read_text_file" above, except the callback // message gives a url from which the file can be // downloaded using standard AJAX. - public read_file(opts: { + read_file = (opts: { project_id: string; // string or array of strings path: string; // string or array of strings compute_server_id?: number; - }): string { + }): string => { const base_path = appBasePath; if (opts.path[0] === "/") { // absolute path to the root @@ -134,10 +116,9 @@ export class ProjectClient { url += `?id=${opts.compute_server_id}`; } return url; - } + }; - public async copy_path_between_projects(opts: { - public?: boolean; // used e.g., by share server landing page action. + copy_path_between_projects = async (opts: { src_project_id: string; // id of source project src_path: string; // relative path of director or file in the source project target_project_id: string; // if of target project @@ -145,36 +126,17 @@ export class ProjectClient { overwrite_newer?: boolean; // overwrite newer versions of file at destination (destructive) delete_missing?: boolean; // delete files in dest that are missing from source (destructive) backup?: boolean; // make ~ backup files instead of overwriting changed files - timeout?: number; // **timeout in seconds** -- how long to wait for the copy to complete before reporting "error" (though it could still succeed) + timeout?: number; // **timeout in milliseconds** -- how long to wait for the copy to complete before reporting "error" (though it could still succeed) exclude?: string[]; // list of patterns to exclude; this uses exactly the (confusing) rsync patterns - }): Promise { - const is_public = opts.public; - delete opts.public; - - if (opts.target_path == null) { - opts.target_path = opts.src_path; - } - - const mesg = is_public - ? message.copy_public_path_between_projects(opts) - : message.copy_path_between_projects(opts); - mesg.wait_until_done = true; // TODO: our UI only supports this for now. - - // THIS CAN BE USEFUL FOR DEBUGGING! - // mesg.debug_delay_s = 10; - - await this.client.async_call({ - timeout: opts.timeout, - message: mesg, - allow_post: false, // since it may take too long - }); - } + }): Promise => { + await this.client.conat_client.hub.projects.copyPathBetweenProjects(opts); + }; // Set a quota parameter for a given project. // As of now, only user in the admin group can make these changes. - public async set_quotas(opts: { + set_quotas = async (opts: { project_id: string; - memory?: number; // see message.js for the units, etc., for all these settings + memory?: number; memory_request?: number; cpu_shares?: number; cores?: number; @@ -183,16 +145,17 @@ export class ProjectClient { network?: number; member_host?: number; always_running?: number; - }): Promise { + }): Promise => { // we do some extra work to ensure all the quotas are numbers (typescript isn't // enough; sometimes client code provides strings, which can cause lots of trouble). const x = coerce_codomain_to_numbers(copy_without(opts, ["project_id"])); - await this.call( - message.project_set_quotas({ ...x, ...{ project_id: opts.project_id } }), - ); - } + await this.client.conat_client.hub.projects.setQuotas({ + ...x, + project_id: opts.project_id, + }); + }; - public async websocket(project_id: string): Promise { + websocket = async (project_id: string): Promise => { const store = redux.getStore("projects"); // Wait until project is running (or admin and not on project) await store.async_wait({ @@ -219,7 +182,7 @@ export class ProjectClient { throw Error("no access to project websocket"); } return await connection_to_project(project_id); - } + }; api = async (project_id: string): Promise => { return (await this.websocket(project_id)).api; @@ -240,7 +203,7 @@ export class ProjectClient { time" (which is stored in the db), which they client will know. This is used, e.g., for operations like "run rst2html on this file whenever it is saved." */ - public async exec(opts: ExecOpts & { post?: boolean }): Promise { + exec = async (opts: ExecOpts & { post?: boolean }): Promise => { if ("async_get" in opts) { opts = defaults(opts, { project_id: required, @@ -288,19 +251,10 @@ export class ProjectClient { }; } - const { post } = opts; - delete opts.post; - try { - let msg; - if (post) { - // use post API - msg = await httpApi("exec", opts); - } else { - const ws = await this.websocket(opts.project_id); - const exec_opts = copy_without(opts, ["project_id"]); - msg = await ws.api.exec(exec_opts); - } + const ws = await this.websocket(opts.project_id); + const exec_opts = copy_without(opts, ["project_id", "cb"]); + const msg = await ws.api.exec(exec_opts); if (msg.status && msg.status == "error") { throw new Error(msg.error); } @@ -333,18 +287,18 @@ export class ProjectClient { }; } } - } + }; // Directly compute the directory listing. No caching or other information // is used -- this just sends a message over the websocket requesting // the backend node.js project process to compute the listing. - public async directory_listing(opts: { + directory_listing = async (opts: { project_id: string; path: string; compute_server_id: number; timeout?: number; hidden?: boolean; - }): Promise<{ files: DirectoryListingEntry[] }> { + }): Promise<{ files: DirectoryListingEntry[] }> => { if (opts.timeout == null) opts.timeout = 15; const api = await this.api(opts.project_id); const listing = await api.listing( @@ -354,9 +308,9 @@ export class ProjectClient { opts.compute_server_id, ); return { files: listing }; - } + }; - public async find_directories(opts: { + find_directories = async (opts: { project_id: string; query?: string; // see the -iwholename option to the UNIX find command. path?: string; // Root path to find directories from @@ -367,7 +321,7 @@ export class ProjectClient { path: string; project_id: string; directories: string[]; - }> { + }> => { opts = defaults(opts, { project_id: required, query: "*", // see the -iwholename option to the UNIX find command. @@ -423,11 +377,11 @@ export class ProjectClient { project_id: opts.project_id, directories: v, }; - } + }; // This is async, so do "await smc_webapp.configuration(...project_id...)". // for reuseInFlight, see https://github.com/sagemathinc/cocalc/issues/7806 - public configuration = reuseInFlight( + configuration = reuseInFlight( async ( project_id: string, aspect: ConfigurationAspect, @@ -440,11 +394,6 @@ export class ProjectClient { }, ); - // Remove all upgrades from all projects that this user collaborates on. - public async remove_all_upgrades(projects?: string[]): Promise { - await this.call(message.remove_all_upgrades({ projects })); - } - touch_project = async ( // project_id where activity occured project_id: string, @@ -487,7 +436,7 @@ export class ProjectClient { } this.touch_throttle[project_id] = Date.now(); try { - await this.client.nats_client.hub.db.touch({ project_id }); + await this.client.conat_client.hub.db.touch({ project_id }); } catch (err) { // silently ignore; this happens, e.g., if you touch too frequently, // and shouldn't be fatal and break other things. @@ -498,36 +447,27 @@ export class ProjectClient { } }; - // Print file to pdf + // Print sagews to pdf // The printed version of the file will be created in the same directory // as path, but with extension replaced by ".pdf". - // Only used for sagews, and would be better done with websocket api anyways... - public async print_to_pdf(opts: { + // Only used for sagews. + print_to_pdf = async ({ + project_id, + path, + options, + timeout, + }: { project_id: string; path: string; - options?: any; // optional options that get passed to the specific backend for this file type timeout?: number; // client timeout -- some things can take a long time to print! - }): Promise { - // returns path to pdf file - if (opts.options == null) opts.options = {}; - opts.options.timeout = opts.timeout; // timeout on backend - - return ( - await this.client.async_call({ - message: message.local_hub({ - project_id: opts.project_id, - message: message.print_to_pdf({ - path: opts.path, - options: opts.options, - }), - }), - timeout: opts.timeout, - allow_post: false, - }) - ).path; - } + options?: any; // optional options that get passed to the specific backend for this file type + }): Promise => { + return await this.client.conat_client + .projectApi({ project_id }) + .editor.printSageWS({ path, timeout, options }); + }; - public async create(opts: { + create = async (opts: { title: string; description: string; image?: string; @@ -536,37 +476,30 @@ export class ProjectClient { license?: string; // never use pool noPool?: boolean; - }): Promise { + }): Promise => { const project_id = - await this.client.nats_client.hub.projects.createProject(opts); + await this.client.conat_client.hub.projects.createProject(opts); this.client.tracking_client.user_tracking("create_project", { project_id, title: opts.title, }); return project_id; - } - - // Disconnect whatever hub we are connected to from the project - // Adding this right now only for debugging/dev purposes! - public async disconnect_hub_from_project(project_id: string): Promise { - await this.call(message.disconnect_from_project({ project_id })); - } + }; - public async realpath(opts: { + realpath = async (opts: { project_id: string; path: string; - }): Promise { - const real = (await this.api(opts.project_id)).realpath(opts.path); - return real; - } + }): Promise => { + return (await this.api(opts.project_id)).realpath(opts.path); + }; - async isdir({ + isdir = async ({ project_id, path, }: { project_id: string; path: string; - }): Promise { + }): Promise => { const { stdout, exit_code } = await this.exec({ project_id, command: "file", @@ -574,40 +507,9 @@ export class ProjectClient { err_on_exit: false, }); return !exit_code && stdout.trim() == "directory"; - } - - // Add and remove a license from a project. Note that these - // might not be used to implement anything in the client frontend, but - // are used via the API, and this is a convenient way to test them. - public async add_license_to_project( - project_id: string, - license_id: string, - ): Promise { - await this.call(message.add_license_to_project({ project_id, license_id })); - } - - public async remove_license_from_project( - project_id: string, - license_id: string, - ): Promise { - await this.call( - message.remove_license_from_project({ project_id, license_id }), - ); - } - - public project_info(project_id: string): ProjectInfo { - return project_info(this.client, project_id); - } - - public project_status(project_id: string): ProjectStatus { - return project_status(this.client, project_id); - } - - public usage_info(project_id: string): UsageInfoWS { - return get_usage_info(project_id); - } + }; - public ipywidgetsGetBuffer = reuseInFlight( + ipywidgetsGetBuffer = reuseInFlight( async ( project_id: string, path: string, @@ -623,21 +525,23 @@ export class ProjectClient { ); // getting, setting, editing, deleting, etc., the api keys for a project - public async api_keys(opts: { + api_keys = async (opts: { project_id: string; action: "get" | "delete" | "create" | "edit"; password?: string; name?: string; id?: number; expire?: Date; - }): Promise { - return await this.client.nats_client.hub.system.manageApiKeys(opts); - } + }): Promise => { + return await this.client.conat_client.hub.system.manageApiKeys(opts); + }; computeServers = (project_id) => { const cs = redux.getProjectActions(project_id)?.computeServers(); if (cs == null) { - throw Error("bug"); + // this happens if something tries to access the compute server info after the project + // tab is closed. It shouldn't do that. + throw Error("compute server information not available"); } return cs; }; @@ -649,9 +553,13 @@ export class ProjectClient { return await this.computeServers(project_id)?.getServerIdForPath(path); }; - // will throw exception if compute servers dkv not yet initialized + // will return undefined if compute servers not yet initialized getServerIdForPathSync = ({ project_id, path }): number | undefined => { - return this.computeServers(project_id).get(path); + const cs = this.computeServers(project_id); + if (cs?.state != "connected") { + return undefined; + } + return cs.get(path); }; } diff --git a/src/packages/frontend/client/purchases.ts b/src/packages/frontend/client/purchases.ts index 12e0357f8b..85ed05ff38 100644 --- a/src/packages/frontend/client/purchases.ts +++ b/src/packages/frontend/client/purchases.ts @@ -32,7 +32,7 @@ export class PurchasesClient { } async getBalance(): Promise { - return await this.client.nats_client.hub.purchases.getBalance(); + return await this.client.conat_client.hub.purchases.getBalance(); } async getSpendRate(): Promise { diff --git a/src/packages/frontend/client/query.ts b/src/packages/frontend/client/query.ts index c5d96539a5..82ebb141ec 100644 --- a/src/packages/frontend/client/query.ts +++ b/src/packages/frontend/client/query.ts @@ -6,15 +6,14 @@ import { is_array } from "@cocalc/util/misc"; import { validate_client_query } from "@cocalc/util/schema-validate"; import { CB } from "@cocalc/util/types/database"; -import { NatsChangefeed } from "@cocalc/sync/table/changefeed-nats2"; +import { ConatChangefeed } from "@cocalc/sync/table/changefeed-conat"; import { uuid } from "@cocalc/util/misc"; -import { client_db } from "@cocalc/util/schema"; declare const $: any; // jQuery export class QueryClient { private client: any; - private changefeeds: { [id: string]: NatsChangefeed } = {}; + private changefeeds: { [id: string]: ConatChangefeed } = {}; constructor(client: any) { this.client = client; @@ -47,7 +46,7 @@ export class QueryClient { } let changefeed; try { - changefeed = new NatsChangefeed({ + changefeed = new ConatChangefeed({ account_id: this.client.account_id, query: opts.query, options: opts.options, @@ -73,19 +72,11 @@ export class QueryClient { if (err) { throw Error(err); } - const query = await this.client.nats_client.hub.db.userQuery({ + const query = await this.client.conat_client.hub.db.userQuery({ query: opts.query, options: opts.options, }); - if (query && !opts.options?.[0]?.["set"]) { - // set thing isn't needed but doesn't hurt - // deal with timestamp versus Date and JSON using our schema. - for (const table in query) { - client_db.processDates({ table, rows: query[table] }); - } - } - if (opts.cb == null) { return { query }; } else { diff --git a/src/packages/frontend/client/stripe.ts b/src/packages/frontend/client/stripe.ts deleted file mode 100644 index 6462047088..0000000000 --- a/src/packages/frontend/client/stripe.ts +++ /dev/null @@ -1,182 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -// COMPLETEY DEPRECATED -- DELETE THIS ? - -/* -stripe payments api via backend hub -*/ - -import { callback2 } from "@cocalc/util/async-utils"; -import * as message from "@cocalc/util/message"; -import { PurchaseInfo } from "@cocalc/util/licenses/purchase/types"; -import { HubClient } from "./hub"; - -export class StripeClient { - private call_api: typeof HubClient.prototype.call; - - constructor(call_api) { - this.call_api = call_api; - } - - private async call(mesg): Promise { - return await callback2(this.call_api, { - message: mesg, - error_event: true, - timeout: 15, - }); - } - - // gets custormer info (if any) and stripe public api key - // for this user, if they are logged in - public async get_customer(): Promise<{ - stripe_publishable_key?: string; - customer: any; - }> { - const mesg = await this.call(message.stripe_get_customer()); - if (mesg == null) { - // evidently this happened -- see - // https://github.com/sagemathinc/cocalc/issues/3711 - throw Error("mesg must be defined"); - } else { - return { - stripe_publishable_key: mesg.stripe_publishable_key, - customer: mesg.customer, - }; - } - } - - public async create_source(token: string): Promise { - return await this.call(message.stripe_create_source({ token })); - } - - public async delete_source(card_id: string): Promise { - return await this.call(message.stripe_delete_source({ card_id })); - } - - public async update_source(card_id: string, info: any): Promise { - return await this.call(message.stripe_update_source({ card_id, info })); - } - - public async set_default_source(card_id: string): Promise { - return await this.call(message.stripe_set_default_source({ card_id })); - } - - // gets list of past stripe charges for this account. - public async get_charges(opts: { - limit?: number; // between 1 and 100 (default: 10) - ending_before?: any; - starting_after?: any; - }): Promise { - return ( - await this.call( - message.stripe_get_charges({ - limit: opts.limit, - ending_before: opts.ending_before, - starting_after: opts.starting_after, - }) - ) - ).charges; - } - - public async create_subscription(opts: { - plan: string; - quantity?: number; - coupon_id?: string; - }): Promise { - if (opts.quantity == null) { - opts.quantity = 1; - } - return await this.call(message.stripe_create_subscription(opts)); - } - - public async cancel_subscription(opts: { - subscription_id: string; - at_period_end?: boolean; // default is *true* - }) { - if (opts.at_period_end == null) { - opts.at_period_end = true; - } - return await this.call(message.stripe_cancel_subscription(opts)); - } - - public async update_subscription(opts: { - subscription_id: string; - quantity?: number; // if given, must be >= number of projects - coupon_id?: string; - projects?: string[]; // ids of projects that subscription applies to (TOD: what?) - plan?: string; - }) { - return await this.call(message.stripe_update_subscription(opts)); - } - - // gets list of past stripe charges for this account. - public async get_subscriptions(opts: { - limit?: number; // between 1 and 100 (default: 10) - ending_before?: any; // see https://stripe.com/docs/api/node#list_subscriptions - starting_after?: any; - }) { - return (await this.call(message.stripe_get_subscriptions(opts))) - .subscriptions; - } - - // Gets the coupon for this account. Returns an error if invalid - // https://stripe.com/docs/api#retrieve_coupon - public async get_coupon(coupon_id: string) { - return (await this.call(message.stripe_get_coupon({ coupon_id }))).coupon; - } - - // gets list of invoices for this account. - public async get_invoices(opts: { - limit?: number; // between 1 and 100 (default: 10) - ending_before?: any; // see https://stripe.com/docs/api/node#list_charges - starting_after?: any; - }) { - return ( - await this.call( - message.stripe_get_invoices({ - limit: opts.limit, - ending_before: opts.ending_before, - starting_after: opts.starting_after, - }) - ) - ).invoices; - } - - public async admin_create_invoice_item( - opts: - | { - account_id: string; - email_address?: string; - amount?: number; // in US dollars -- if amount/description *not* given, then merely ensures user has stripe account and updats info about them - description?: string; - } - | { - account_id?: string; - email_address: string; - amount?: number; - description?: string; - } - ) { - return await this.call(message.stripe_admin_create_invoice_item(opts)); - } - - // Make it so the SMC user with the given email address has a corresponding stripe - // identity, even if they have never entered a credit card. May only be used by - // admin users. - public async admin_create_customer( - opts: - | { account_id: string; email_address?: string } - | { account_id?: string; email_address: string } - ) { - return await this.admin_create_invoice_item(opts); - } - - // Purchase a license (technically this may or may not involve "stripe"). - public async purchase_license(info: PurchaseInfo): Promise { - return (await this.call(message.purchase_license({ info }))).resp; - } - -} diff --git a/src/packages/frontend/client/time.ts b/src/packages/frontend/client/time.ts index 76b4ce3e99..6164349c29 100644 --- a/src/packages/frontend/client/time.ts +++ b/src/packages/frontend/client/time.ts @@ -3,7 +3,7 @@ * License: MS-RSL – see LICENSE.md for details */ -import getTime, { getLastSkew, getLastPingTime } from "@cocalc/nats/time"; +import getTime, { getLastSkew, getLastPingTime } from "@cocalc/conat/time"; const PING_INTERVAL_MS = 10000; @@ -29,7 +29,7 @@ export class TimeClient { this.closed = true; } - // everything related to sync should directly use nats' getTime, which + // everything related to sync should directly use conat getTime, which // throws an error if it doesn't know the correct server time. server_time = (): Date => { try { diff --git a/src/packages/frontend/client/tracking.ts b/src/packages/frontend/client/tracking.ts index 3a8dc2b13b..9b376ac1e3 100644 --- a/src/packages/frontend/client/tracking.ts +++ b/src/packages/frontend/client/tracking.ts @@ -4,7 +4,6 @@ */ import { WebappClient } from "./client"; -import * as message from "@cocalc/util/message"; import { redux } from "@cocalc/frontend/app-framework"; export class TrackingClient { @@ -23,7 +22,7 @@ export class TrackingClient { ?.get("user_tracking"); } if (this.userTrackingEnabled == "yes") { - await this.client.nats_client.hub.system.userTracking({ event, value }); + await this.client.conat_client.hub.system.userTracking({ event, value }); } }; @@ -36,12 +35,19 @@ export class TrackingClient { return; } this.log_error_cache[error] = Date.now(); - this.client.call({ - message: message.log_client_error({ error }), - }); + (async () => { + try { + await this.client.conat_client.hub.system.logClientError({ + event: "error", + error, + }); + } catch (err) { + console.log(`WARNING -- issue reporting error -- ${err}`); + } + })(); }; webapp_error = async (opts: object): Promise => { - await this.client.async_call({ message: message.webapp_error(opts) }); + await this.client.conat_client.hub.system.webappError(opts); }; } diff --git a/src/packages/frontend/client/users.ts b/src/packages/frontend/client/users.ts index 4598ef9e7e..75e1cbf01a 100644 --- a/src/packages/frontend/client/users.ts +++ b/src/packages/frontend/client/users.ts @@ -50,7 +50,7 @@ export class UsersClient { admin?: boolean; // admins can do an admin version of the query, which also does substring searches on email address (not just name) only_email?: boolean; // search only via email address }): Promise => { - return await this.client.nats_client.hub.system.userSearch({ + return await this.client.conat_client.hub.system.userSearch({ query, limit, admin, @@ -99,7 +99,7 @@ export class UsersClient { } } if (v.length > 0) { - const names = await this.client.nats_client.hub.system.getNames(v); + const names = await this.client.conat_client.hub.system.getNames(v); for (const account_id of v) { // iterate over v to record accounts that don't exist too x[account_id] = names[account_id]; diff --git a/src/packages/frontend/collaborators/project-invite-tokens.tsx b/src/packages/frontend/collaborators/project-invite-tokens.tsx index f1e590fabe..83a89de303 100644 --- a/src/packages/frontend/collaborators/project-invite-tokens.tsx +++ b/src/packages/frontend/collaborators/project-invite-tokens.tsx @@ -123,7 +123,7 @@ export const ProjectInviteTokens: React.FC = React.memo( return heading; } - async function add_token(expires) { + async function add_token(expires: Date) { if (tokens != null && tokens.length > MAX_TOKENS) { // TODO: just in case of some weird abuse... and until we implement // deletion of tokens. Maybe the backend will just purge @@ -136,7 +136,6 @@ export const ProjectInviteTokens: React.FC = React.memo( return; } const token = secure_random_token(TOKEN_LENGTH); - try { await webapp_client.async_query({ query: { @@ -144,7 +143,7 @@ export const ProjectInviteTokens: React.FC = React.memo( token, project_id, created: webapp_client.server_time(), - expires: expires, + expires, }, }, }); @@ -187,7 +186,7 @@ export const ProjectInviteTokens: React.FC = React.memo( const handleModalOK = () => { // const name = form.getFieldValue("name"); const expire = form.getFieldValue("expire"); - add_token(expire); + add_token(expire.toDate()); setAddModalVisible(false); form.resetFields(); }; diff --git a/src/packages/frontend/components/loading.tsx b/src/packages/frontend/components/loading.tsx index 943aafcf40..8e0dac0769 100644 --- a/src/packages/frontend/components/loading.tsx +++ b/src/packages/frontend/components/loading.tsx @@ -5,7 +5,7 @@ import { CSSProperties } from "react"; import { useIntl } from "react-intl"; - +import FakeProgress from "@cocalc/frontend/components/fake-progress"; import { TypedMap, useDelayedRender } from "@cocalc/frontend/app-framework"; import { labels } from "@cocalc/frontend/i18n"; import { Icon } from "./icon"; @@ -63,7 +63,9 @@ export function Loading({ {text ?? intl.formatMessage(labels.loading)} {estimate != undefined && ( -
    Estimated time: {estimate.get("time")}s
    +
    + +
    )}
    ); diff --git a/src/packages/frontend/compute/manager.ts b/src/packages/frontend/compute/manager.ts deleted file mode 100644 index 25cc99824a..0000000000 --- a/src/packages/frontend/compute/manager.ts +++ /dev/null @@ -1,35 +0,0 @@ -/* -Client side compute servers manager - -Used from a browser client frontend to manage what compute servers -are available and how they are used for a given project. - -When doing dev from the browser console, do: - - cc.client.project_client.computeServers(cc.current().project_id) -*/ - -import { - computeServerManager, - type ComputeServerManager, -} from "@cocalc/nats/compute/manager"; - -const computeServerManagerCache: { - [project_id: string]: ComputeServerManager; -} = {}; - -// very simple cache with no ref counting or anything. -// close a manager only when closing the project. -export default function computeServers( - project_id: string, -): ComputeServerManager { - if (computeServerManagerCache[project_id]) { - return computeServerManagerCache[project_id]; - } - const M = computeServerManager({ project_id }); - computeServerManagerCache[project_id] = M; - M.on("closed", () => { - delete computeServerManagerCache[project_id]; - }); - return M; -} diff --git a/src/packages/frontend/compute/select-server-for-file.tsx b/src/packages/frontend/compute/select-server-for-file.tsx index 46173322d9..3376b161cf 100644 --- a/src/packages/frontend/compute/select-server-for-file.tsx +++ b/src/packages/frontend/compute/select-server-for-file.tsx @@ -40,7 +40,7 @@ export default function SelectComputeServerForFile({ if (frame_id == null) { throw Error("frame_id is required for terminal"); } - return actions.terminals.get(frame_id)?.term_path; + return actions.terminals.get(frame_id)?.termPath; } if (type == "chat") { return chatFile(path); diff --git a/src/packages/frontend/compute/serial-port-output.tsx b/src/packages/frontend/compute/serial-port-output.tsx index 00bc325459..55c1d66613 100644 --- a/src/packages/frontend/compute/serial-port-output.tsx +++ b/src/packages/frontend/compute/serial-port-output.tsx @@ -18,7 +18,7 @@ const EXPONENTIAL_BACKOFF = 1.3; import { Button, Checkbox, Modal, Spin, Tooltip } from "antd"; import { useCallback, useEffect, useRef, useState } from "react"; import ReactDOM from "react-dom"; -import { Terminal } from "xterm"; +import { Terminal } from "@xterm/xterm"; import { redux } from "@cocalc/frontend/app-framework"; import { Icon } from "@cocalc/frontend/components"; diff --git a/src/packages/frontend/conat/client.ts b/src/packages/frontend/conat/client.ts new file mode 100644 index 0000000000..2f1ae0db2c --- /dev/null +++ b/src/packages/frontend/conat/client.ts @@ -0,0 +1,500 @@ +import { redux } from "@cocalc/frontend/app-framework"; +import type { WebappClient } from "@cocalc/frontend/client/client"; +import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; +import { + type ConatSyncTable, + ConatSyncTableFunction, +} from "@cocalc/conat/sync/synctable"; +import { randomId, inboxPrefix } from "@cocalc/conat/names"; +import { projectSubject } from "@cocalc/conat/names"; +import { parseQueryWithOptions } from "@cocalc/sync/table/util"; +import { type HubApi, initHubApi } from "@cocalc/conat/hub/api"; +import { type ProjectApi, initProjectApi } from "@cocalc/conat/project/api"; +import { isValidUUID } from "@cocalc/util/misc"; +import { createOpenFiles, OpenFiles } from "@cocalc/conat/sync/open-files"; +import { PubSub } from "@cocalc/conat/sync/pubsub"; +import type { ChatOptions } from "@cocalc/util/types/llm"; +import { dkv } from "@cocalc/conat/sync/dkv"; +import { akv } from "@cocalc/conat/sync/akv"; +import { astream } from "@cocalc/conat/sync/astream"; +import { dko } from "@cocalc/conat/sync/dko"; +import { dstream } from "@cocalc/conat/sync/dstream"; +import { callConatService, createConatService } from "@cocalc/conat/service"; +import type { + CallConatServiceFunction, + CreateConatServiceFunction, +} from "@cocalc/conat/service"; +import { listingsClient } from "@cocalc/conat/service/listings"; +import getTime, { getSkew, init as initTime } from "@cocalc/conat/time"; +import { llm } from "@cocalc/conat/llm/client"; +import { inventory } from "@cocalc/conat/sync/inventory"; +import { EventEmitter } from "events"; +import { + getClient as getClientWithState, + setConatClient, + type ClientWithState, +} from "@cocalc/conat/client"; +import Cookies from "js-cookie"; +import { ACCOUNT_ID_COOKIE } from "@cocalc/frontend/client/client"; +import { info as refCacheInfo } from "@cocalc/util/refcache"; +import { connect as connectToConat } from "@cocalc/conat/core/client"; +import type { ConnectionStats } from "@cocalc/conat/core/types"; +import { appBasePath } from "@cocalc/frontend/customize/app-base-path"; +import { once } from "@cocalc/util/async-utils"; +import { delay } from "awaiting"; +import { + deleteRememberMe, + setRememberMe, +} from "@cocalc/frontend/misc/remember-me"; + +export interface ConatConnectionStatus { + state: "connected" | "disconnected"; + reason: string; + details: any; + stats: ConnectionStats; +} + +const DEFAULT_TIMEOUT = 15000; + +const DEBUG = false; + +export class ConatClient extends EventEmitter { + client: WebappClient; + public hub: HubApi; + public sessionId = randomId(); + private openFilesCache: { [project_id: string]: OpenFiles } = {}; + private clientWithState: ClientWithState; + private _conatClient: null | ReturnType; + public numConnectionAttempts = 0; + + constructor(client: WebappClient) { + super(); + this.setMaxListeners(100); + this.client = client; + this.hub = initHubApi(this.callHub); + this.initConatClient(); + this.on("state", (state) => { + this.emit(state); + }); + } + + private setConnectionStatus = (status: Partial) => { + const actions = redux?.getActions("page"); + const store = redux?.getStore("page"); + if (actions == null || store == null) { + return; + } + const cur = store.get("conat")?.toJS(); + actions.setState({ conat: { ...cur, ...status } } as any); + }; + + conat = () => { + if (this._conatClient == null) { + this.startStatsReporter(); + const address = location.origin + appBasePath; + this._conatClient = connectToConat({ + address, + inboxPrefix: inboxPrefix({ account_id: this.client.account_id }), + }); + this._conatClient.on("connected", () => { + this.setConnectionStatus({ + state: "connected", + reason: "", + details: "", + stats: this._conatClient?.stats, + }); + this.client.emit("connected"); + }); + this._conatClient.on("disconnected", (reason, details) => { + this.setConnectionStatus({ + state: "disconnected", + reason, + details, + stats: this._conatClient?.stats, + }); + this.client.emit("disconnected", "offline"); + }); + this._conatClient.conn.io.on("reconnect_attempt", (attempt) => { + this.numConnectionAttempts = attempt; + this.client.emit("connecting"); + }); + } + return this._conatClient!; + }; + + private permanentlyDisconnected = false; + permanentlyDisconnect = () => { + this.permanentlyDisconnected = true; + this.standby(); + }; + + is_signed_in = (): boolean => { + return !!this._conatClient?.info?.user?.account_id; + }; + + is_connected = (): boolean => { + return !!this._conatClient?.conn?.connected; + }; + + private startStatsReporter = async () => { + while (true) { + if (this._conatClient != null) { + this.setConnectionStatus({ stats: this._conatClient?.stats }); + } + await delay(5000); + } + }; + + private initConatClient = async () => { + setConatClient({ + account_id: this.client.account_id, + conat: this.conat, + reconnect: async () => this.reconnect(), + getLogger: DEBUG + ? (name) => { + return { + info: (...args) => console.info(name, ...args), + debug: (...args) => console.log(name, ...args), + warn: (...args) => console.warn(name, ...args), + silly: (...args) => console.log(name, ...args), + }; + } + : undefined, + }); + this.clientWithState = getClientWithState(); + this.clientWithState.on("state", (state) => { + if (state != "closed") { + this.emit(state); + } + }); + initTime(); + const client = this.conat(); + if (!client.info) { + await once(client.conn as any, "info"); + } + if (client.info?.user?.account_id) { + console.log("Connected as ", JSON.stringify(client.info?.user)); + this.signedIn({ + account_id: client.info.user.account_id, + hub: client.info.id, + }); + const cookie = Cookies.get(ACCOUNT_ID_COOKIE); + if (cookie && cookie != client.info.user.account_id) { + // make sure account_id cookie is set to the actual account we're + // signed in as, then refresh since some things are going to be + // broken otherwise. To test this use dev tools and just change the account_id + // cookies value to something random. + Cookies.set(ACCOUNT_ID_COOKIE, client.info.user.account_id); + // and we're out of here: + location.reload(); + } + } else { + console.log("Sign in failed -- ", client.info); + this.signInFailed(client.info?.user?.error ?? "Failed to sign in."); + this.client.alert_message({ + type: "error", + message: client.info?.user?.error, + block: true, + }); + this.standby(); + } + }; + + public signedInMessage?: { account_id: string; hub: string }; + private signedIn = (mesg: { account_id: string; hub: string }) => { + this.signedInMessage = mesg; + this.client.account_id = mesg.account_id; + setRememberMe(appBasePath); + this.client.emit("signed_in", mesg); + }; + + private signInFailed = (error) => { + deleteRememberMe(appBasePath); + this.client.emit("remember_me_failed", { error }); + }; + + reconnect = () => { + this._conatClient?.conn.io.engine.close(); + this.resume(); + }; + + // if there is a connection, put it in standby + standby = () => { + // @ts-ignore + this._conatClient?.disconnect(); + }; + + // if there is a connection, resume it + resume = () => { + if (this.permanentlyDisconnected) { + console.log( + "Not connecting -- client is permanently disconnected and must refresh their browser", + ); + return; + } + this._conatClient?.conn.io.connect(); + }; + + callConatService: CallConatServiceFunction = async (options) => { + return await callConatService(options); + }; + + createConatService: CreateConatServiceFunction = (options) => { + return createConatService(options); + }; + + projectWebsocketApi = async ({ + project_id, + compute_server_id, + mesg, + timeout = DEFAULT_TIMEOUT, + }) => { + const cn = this.conat(); + const subject = projectSubject({ + project_id, + compute_server_id, + service: "browser-api", + }); + const resp = await cn.request(subject, mesg, { + timeout, + waitForInterest: true, + }); + return resp.data; + }; + + private callHub = async ({ + service = "api", + name, + args = [], + timeout = DEFAULT_TIMEOUT, + }: { + service?: string; + name: string; + args: any[]; + timeout?: number; + }) => { + const cn = this.conat(); + const subject = `hub.account.${this.client.account_id}.${service}`; + try { + const data = { name, args }; + const resp = await cn.request(subject, data, { timeout }); + return resp.data; + } catch (err) { + err.message = `${err.message} - callHub: subject='${subject}', name='${name}', `; + throw err; + } + }; + + // Returns api for RPC calls to the project with typescript support! + // if compute_server_id is NOT given then: + // if path is given use compute server id for path (assuming mapping is loaded) + // if path is not given, use current project default + projectApi = ({ + project_id, + compute_server_id, + path, + timeout = DEFAULT_TIMEOUT, + }: { + project_id: string; + path?: string; + compute_server_id?: number; + // IMPORTANT: this timeout is only AFTER user is connected. + timeout?: number; + }): ProjectApi => { + if (!isValidUUID(project_id)) { + throw Error(`project_id = '${project_id}' must be a valid uuid`); + } + if (compute_server_id == null) { + const actions = redux.getProjectActions(project_id); + if (path != null) { + compute_server_id = + actions.getComputeServerIdForFile({ path }) ?? + actions.getComputeServerId(); + } else { + compute_server_id = actions.getComputeServerId(); + } + } + const callProjectApi = async ({ name, args }) => { + return await this.callProject({ + project_id, + compute_server_id, + timeout, + service: "api", + name, + args, + }); + }; + return initProjectApi(callProjectApi); + }; + + private callProject = async ({ + service = "api", + project_id, + compute_server_id, + name, + args = [], + timeout = DEFAULT_TIMEOUT, + }: { + service?: string; + project_id: string; + compute_server_id?: number; + name: string; + args: any[]; + timeout?: number; + }) => { + const cn = this.conat(); + const subject = projectSubject({ project_id, compute_server_id, service }); + const resp = await cn.request( + subject, + { name, args }, + // we use waitForInterest because often the project hasn't + // quite fully started. + { timeout, waitForInterest: true }, + ); + return resp.data; + }; + + synctable: ConatSyncTableFunction = async ( + query0, + options?, + ): Promise => { + const { query, table } = parseQueryWithOptions(query0, options); + if (options?.project_id != null && query[table][0]["project_id"] === null) { + query[table][0]["project_id"] = options.project_id; + } + return await this.conat().sync.synctable({ + ...options, + query, + account_id: this.client.account_id, + }); + }; + + primus = ({ + project_id, + compute_server_id = 0, + channel, + }: { + project_id: string; + compute_server_id?: number; + channel?: string; + }) => { + let subject = projectSubject({ + project_id, + compute_server_id, + service: "primus", + }); + if (channel) { + subject += "." + channel; + } + return this.conat().socket.connect(subject, { + desc: `primus-${channel ?? ""}`, + }); + }; + + openFiles = reuseInFlight(async (project_id: string) => { + if (this.openFilesCache[project_id] == null) { + const openFiles = await createOpenFiles({ + project_id, + }); + this.openFilesCache[project_id] = openFiles; + openFiles.on("closed", () => { + delete this.openFilesCache[project_id]; + }); + openFiles.on("change", (entry) => { + if (entry.deleted?.deleted) { + setDeleted({ + project_id, + path: entry.path, + deleted: entry.deleted.time, + }); + } else { + setNotDeleted({ project_id, path: entry.path }); + } + }); + const recentlyDeletedPaths: any = {}; + for (const { path, deleted } of openFiles.getAll()) { + if (deleted?.deleted) { + recentlyDeletedPaths[path] = deleted.time; + } + } + const store = redux.getProjectStore(project_id); + store.setState({ recentlyDeletedPaths }); + } + return this.openFilesCache[project_id]!; + }); + + closeOpenFiles = (project_id) => { + this.openFilesCache[project_id]?.close(); + }; + + pubsub = async ({ + project_id, + path, + name, + }: { + project_id: string; + path?: string; + name: string; + }) => { + return new PubSub({ client: this.conat(), project_id, path, name }); + }; + + // Evaluate an llm. This streams the result if stream is given an option, + // AND it also always returns the result. + llm = async (opts: ChatOptions): Promise => { + return await llm({ account_id: this.client.account_id, ...opts }); + }; + + dstream = dstream; + astream = astream; + dkv = dkv; + akv = akv; + dko = dko; + + listings = async (opts: { + project_id: string; + compute_server_id?: number; + }) => { + return await listingsClient(opts); + }; + + getTime = (): number => { + return getTime(); + }; + + getSkew = async (): Promise => { + return await getSkew(); + }; + + inventory = async (location: { + account_id?: string; + project_id?: string; + }) => { + const inv = await inventory(location); + // @ts-ignore + if (console.log_original != null) { + const ls_orig = inv.ls; + // @ts-ignore + inv.ls = (opts) => ls_orig({ ...opts, log: console.log_original }); + } + return inv; + }; + + refCacheInfo = () => refCacheInfo(); +} + +function setDeleted({ project_id, path, deleted }) { + if (!redux.hasProjectStore(project_id)) { + return; + } + const actions = redux.getProjectActions(project_id); + actions.setRecentlyDeleted(path, deleted); +} + +function setNotDeleted({ project_id, path }) { + if (!redux.hasProjectStore(project_id)) { + return; + } + const actions = redux.getProjectActions(project_id); + actions?.setRecentlyDeleted(path, 0); +} diff --git a/src/packages/frontend/nats/listings.ts b/src/packages/frontend/conat/listings.ts similarity index 95% rename from src/packages/frontend/nats/listings.ts rename to src/packages/frontend/conat/listings.ts index 2d08285bfa..c2c625cfe4 100644 --- a/src/packages/frontend/nats/listings.ts +++ b/src/packages/frontend/conat/listings.ts @@ -15,10 +15,10 @@ import { createListingsApiClient, type ListingsApi, MIN_INTEREST_INTERVAL_MS, -} from "@cocalc/nats/service/listings"; +} from "@cocalc/conat/service/listings"; import { delay } from "awaiting"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; -import { getLogger } from "@cocalc/nats/client"; +import { getLogger } from "@cocalc/conat/client"; const logger = getLogger("listings"); @@ -46,7 +46,6 @@ export class Listings extends EventEmitter { } private createClient = async () => { - await webapp_client.nats_client.addProjectPermissions([this.project_id]); let d = 3000; const MAX_DELAY_MS = 15000; while (this.state != "closed") { @@ -106,12 +105,15 @@ export class Listings extends EventEmitter { await this.listingsClient.watch(path, force); return; } catch (err) { + if (this.listingsClient == null) { + return; + } force = true; logger.debug( `WARNING: not yet able to watch '${path}' in ${this.project_id} -- ${err}`, ); try { - await this.listingsClient.api.nats.waitFor({ + await this.listingsClient.api.conat.waitFor({ maxWait: 7.5 * 1000 * 60, }); } catch (err) { @@ -184,7 +186,7 @@ export class Listings extends EventEmitter { path: string, ): Promise => { if (this.listingsClient == null) { - throw Error("listings not ready"); + return; } return this.listingsClient.get(path)?.files; }; @@ -194,14 +196,15 @@ export class Listings extends EventEmitter { path: string, ): Promise => { if (this.listingsClient == null) { - throw Error("listings not ready"); + // throw Error("listings not ready"); + return; } return this.listingsClient.get(path)?.more ? 1 : 0; }; getMissing = (path: string): number | undefined => { if (this.listingsClient == null) { - throw Error("listings not ready"); + return; } return this.listingsClient.get(path)?.more ? 1 : 0; }; diff --git a/src/packages/frontend/nats/types.ts b/src/packages/frontend/conat/types.ts similarity index 100% rename from src/packages/frontend/nats/types.ts rename to src/packages/frontend/conat/types.ts diff --git a/src/packages/frontend/nats/use-listing.ts b/src/packages/frontend/conat/use-listing.ts similarity index 94% rename from src/packages/frontend/nats/use-listing.ts rename to src/packages/frontend/conat/use-listing.ts index c6d46fc16b..d851b1454c 100644 --- a/src/packages/frontend/nats/use-listing.ts +++ b/src/packages/frontend/conat/use-listing.ts @@ -4,7 +4,7 @@ React Hook to provide access to directory listings in a project. This is NOT used yet, but seems like the right way to do directly listings in a modern clean dynamic way. It would be used like this: -import useListing from "@cocalc/frontend/nats/use-listing"; +import useListing from "@cocalc/frontend/conat/use-listing"; function ListingTest({ path, compute_server_id }) { const listing = useListing({ path, compute_server_id }); return
    {JSON.stringify(listing)}
    ; @@ -17,7 +17,7 @@ import { listingsClient, type ListingsClient, type Listing, -} from "@cocalc/nats/service/listings"; +} from "@cocalc/conat/service/listings"; import { useAsyncEffect } from "use-async-effect"; import { useProjectContext } from "@cocalc/frontend/project/context"; diff --git a/src/packages/frontend/course/assignments/actions.ts b/src/packages/frontend/course/assignments/actions.ts index c45160d01d..a50934da00 100644 --- a/src/packages/frontend/course/assignments/actions.ts +++ b/src/packages/frontend/course/assignments/actions.ts @@ -464,7 +464,7 @@ export class AssignmentsActions { overwrite_newer: true, backup: true, delete_missing: false, - timeout: COPY_TIMEOUT_MS / 1000, + timeout: COPY_TIMEOUT_MS, }); // write their name to a file const name = store.get_student_name_extra(student_id); @@ -624,7 +624,7 @@ ${details} backup: true, delete_missing: false, exclude: peer_graded ? ["*GRADER*.txt"] : undefined, - timeout: COPY_TIMEOUT_MS / 1000, + timeout: COPY_TIMEOUT_MS, }); finish(""); } catch (err) { @@ -860,7 +860,7 @@ ${details} overwrite_newer: !!overwrite, // default is "false" delete_missing: !!overwrite, // default is "false" backup: !!!overwrite, // default is "true" - timeout: COPY_TIMEOUT_MS / 1000, + timeout: COPY_TIMEOUT_MS, }; await webapp_client.project_client.copy_path_between_projects(opts); await this.course_actions.compute.setComputeServerAssociations({ @@ -1378,7 +1378,7 @@ ${details} overwrite_newer: false, delete_missing: false, exclude: ["*STUDENT*.txt", "*" + DUE_DATE_FILENAME + "*"], - timeout: COPY_TIMEOUT_MS / 1000, + timeout: COPY_TIMEOUT_MS, }); }; @@ -1461,7 +1461,7 @@ ${details} target_path, overwrite_newer: false, delete_missing: false, - timeout: COPY_TIMEOUT_MS / 1000, + timeout: COPY_TIMEOUT_MS, }); // write local file identifying the grader @@ -2016,7 +2016,7 @@ ${details} overwrite_newer: true, delete_missing: true, backup: false, - timeout: COPY_TIMEOUT_MS / 1000, + timeout: COPY_TIMEOUT_MS, }); } else { ephemeralGradePath = false; diff --git a/src/packages/frontend/course/common/student-assignment-info.tsx b/src/packages/frontend/course/common/student-assignment-info.tsx index 68e9733975..adbf682501 100644 --- a/src/packages/frontend/course/common/student-assignment-info.tsx +++ b/src/packages/frontend/course/common/student-assignment-info.tsx @@ -15,7 +15,6 @@ import { MarkdownInput } from "@cocalc/frontend/editors/markdown-input"; import { labels } from "@cocalc/frontend/i18n"; import { NotebookScores } from "@cocalc/frontend/jupyter/nbgrader/autograde"; import { webapp_client } from "@cocalc/frontend/webapp-client"; -import { to_json } from "@cocalc/util/misc"; import { BigTime } from "."; import { CourseActions } from "../actions"; import { NbgraderScores } from "../nbgrader/scores"; @@ -469,7 +468,7 @@ export function StudentAssignmentInfo({ function render_error(step: Steps, error) { if (typeof error !== "string") { - error = to_json(error); + error = `${error}`; } // We search for two different error messages, since different errors happen in // KuCalc versus other places cocalc runs. It depends on what is doing the copy. diff --git a/src/packages/frontend/course/export/file-use-times.ts b/src/packages/frontend/course/export/file-use-times.ts index 28d4067866..f9aa5eae8b 100644 --- a/src/packages/frontend/course/export/file-use-times.ts +++ b/src/packages/frontend/course/export/file-use-times.ts @@ -37,7 +37,7 @@ async function one_student_file_use_times( const times: { [path: string]: PathUseTimes } = {}; for (const path of paths) { const { edit_times, access_times } = - await webapp_client.nats_client.hub.db.fileUseTimes({ + await webapp_client.conat_client.hub.db.fileUseTimes({ project_id, path, target_account_id: account_id, diff --git a/src/packages/frontend/course/handouts/actions.ts b/src/packages/frontend/course/handouts/actions.ts index fd2b27fd8e..302e074bab 100644 --- a/src/packages/frontend/course/handouts/actions.ts +++ b/src/packages/frontend/course/handouts/actions.ts @@ -260,7 +260,7 @@ export class HandoutsActions { overwrite_newer: !!overwrite, // default is "false" delete_missing: !!overwrite, // default is "false" backup: !!!overwrite, // default is "true" - timeout: COPY_TIMEOUT_MS / 1000, + timeout: COPY_TIMEOUT_MS, }; await webapp_client.project_client.copy_path_between_projects(opts); diff --git a/src/packages/frontend/customize.tsx b/src/packages/frontend/customize.tsx index 02adb9fd0c..5710036c10 100644 --- a/src/packages/frontend/customize.tsx +++ b/src/packages/frontend/customize.tsx @@ -60,6 +60,10 @@ import * as theme from "@cocalc/util/theme"; import { CustomLLMPublic } from "@cocalc/util/types/llm"; import { DefaultQuotaSetting, Upgrades } from "@cocalc/util/upgrades/quota"; export { TermsOfService } from "@cocalc/frontend/customize/terms-of-service"; +import { delay } from "awaiting"; + +// update every 2 minutes. +const UPDATE_INTERVAL = 2 * 60000; // this sets UI modes for using a kubernetes based back-end // 'yes' (historic value) equals 'cocalc.com' @@ -257,6 +261,10 @@ export class CustomizeActions extends Actions { unlicensed_project_timetravel_limit: undefined, }); }; + + reload = async () => { + await loadCustomizeState(); + }; } export const store = redux.createStore("customize", CustomizeStore, defaults); @@ -268,10 +276,7 @@ actions.setState({ is_commercial: true, ssh_gateway: true }); // to generate static content, which can't be customized. export let commercial: boolean = defaults.is_commercial; -// For now, hopefully not used (this was the old approach). -// in the future we might want to reload the configuration, though. -// Note that this *is* clearly used as a fallback below though...! -async function init_customize() { +async function loadCustomizeState() { if (typeof process != "undefined") { // running in node.js return; @@ -312,7 +317,12 @@ async function init_customize() { actions.setState({ token: !!registration }); } -init_customize(); +export async function init() { + while (true) { + await loadCustomizeState(); + await delay(UPDATE_INTERVAL); + } +} function process_ollama(ollama?) { if (!ollama) return; @@ -335,7 +345,7 @@ function process_customize(obj) { for (const k in site_settings_conf) { const v = site_settings_conf[k]; obj[k] = - obj[k] != null ? obj[k] : v.to_val?.(v.default, obj_orig) ?? v.default; + obj[k] != null ? obj[k] : (v.to_val?.(v.default, obj_orig) ?? v.default); } // the llm markup special case obj.llm_markup = obj_orig._llm_markup ?? 30; diff --git a/src/packages/frontend/editor.coffee b/src/packages/frontend/editor.coffee index 2d3b9b7817..93e1e07406 100644 --- a/src/packages/frontend/editor.coffee +++ b/src/packages/frontend/editor.coffee @@ -1094,7 +1094,7 @@ class CodeMirrorEditor extends FileEditor async.series([show_dialog, convert], (err) => if err - msg = "problem printing -- #{misc.to_json(err)}" + msg = "problem printing -- #{err.message}" alert_message type : "error" message : msg @@ -1191,7 +1191,7 @@ class CodeMirrorEditor extends FileEditor dialog.find(".btn-submit").icon_spin(false) dialog.find(".webapp-file-printing-progress").hide() if err - alert_message(type:"error", message:"problem printing '#{p.tail}' -- #{misc.to_json(err)}") + alert_message(type:"error", message:"problem printing '#{p.tail}' -- #{err.message}") ) return false diff --git a/src/packages/frontend/embed/index.ts b/src/packages/frontend/embed/index.ts index 3e492c8d3f..2884acae90 100644 --- a/src/packages/frontend/embed/index.ts +++ b/src/packages/frontend/embed/index.ts @@ -21,6 +21,7 @@ import { init as initApp } from "../app/init"; import { init as initProjects } from "../projects"; import { init as initMarkdown } from "../markdown/markdown-input/main"; import { init as initCrashBanner } from "../crash-banner"; +import { init as initCustomize } from "../customize"; // Do not delete this without first looking at https://github.com/sagemathinc/cocalc/issues/5390 @@ -36,6 +37,7 @@ export async function init() { initApp(); initProjects(); initMarkdown(); + initCustomize(); initLast(); try { await render(); diff --git a/src/packages/frontend/entry-point.ts b/src/packages/frontend/entry-point.ts index 62421f3643..933fc6f0e6 100644 --- a/src/packages/frontend/entry-point.ts +++ b/src/packages/frontend/entry-point.ts @@ -39,6 +39,7 @@ import { init as initMarkdown } from "./markdown/markdown-input/main"; // only enable iframe comms in minimal kiosk mode import { init as initIframeComm } from "./iframe-communication"; import { init as initCrashBanner } from "./crash-banner"; +import { init as initCustomize } from "./customize"; // Do not delete this without first looking at https://github.com/sagemathinc/cocalc/issues/5390 // This import of codemirror forces the initial full load of codemirror @@ -62,6 +63,7 @@ export async function init() { initNotifications(redux); } initMarkdown(); + initCustomize(); if (COCALC_MINIMAL) { initIframeComm(); } diff --git a/src/packages/frontend/file-associations.ts b/src/packages/frontend/file-associations.ts index 3e6bf4d71d..10571fc19b 100644 --- a/src/packages/frontend/file-associations.ts +++ b/src/packages/frontend/file-associations.ts @@ -46,6 +46,7 @@ const codemirror_associations: { [ext: string]: string } = { cpp: "text/x-c++src", cc: "text/x-c++src", tcc: "text/x-c++src", + cjs: "javascript", conf: "nginx", // should really have a list of different types that end in .conf and autodetect based on heuristics, letting user change. csharp: "text/x-csharp", "c#": "text/x-csharp", @@ -80,7 +81,6 @@ const codemirror_associations: { [ext: string]: string } = { jsx: "jsx", json: "javascript", jsonl: "javascript", // See https://jsonlines.org/ - lean: "lean", // obviously nowhere close... ls: "text/x-livescript", lua: "lua", m: "text/x-octave", @@ -271,14 +271,6 @@ file_associations["html"] = { name: "html", } as const; -file_associations["lean"] = { - editor: "lean", // so frame-editors/code-editor won't try to register the lean extension. - icon: "file-code", - opts: { indent_unit: 4, tab_size: 4, mode: "lean" }, - name: "lean", - exclude_from_compute_server: true, -}; - file_associations["md"] = file_associations["markdown"] = { icon: "markdown", opts: { diff --git a/src/packages/frontend/frame-editors/code-editor/actions.ts b/src/packages/frontend/frame-editors/code-editor/actions.ts index ce1442123e..836d298d1e 100644 --- a/src/packages/frontend/frame-editors/code-editor/actions.ts +++ b/src/packages/frontend/frame-editors/code-editor/actions.ts @@ -109,6 +109,7 @@ import * as cm_doc_cache from "./doc"; import { SHELLS } from "./editor"; import { test_line } from "./simulate_typing"; import { misspelled_words } from "./spell-check"; +import { log_opened_time } from "@cocalc/frontend/project/open-file"; interface gutterMarkerParams { line: number; @@ -348,6 +349,11 @@ export class Actions< } this._syncstring.once("ready", (err) => { + if (this.doctype != "none") { + // doctype = 'none' must be handled elsewhere, e.g., terminals. + log_opened_time(this.project_id, this.path); + } + if (err) { this.set_error(`${err}\nFix this, then try opening the file again.`); return; @@ -356,6 +362,7 @@ export class Actions< // the doc could perhaps be closed by the time this init is fired, in which case just bail -- no point in trying to initialize anything. return; } + this._syncstring_init = true; this._syncstring_metadata(); this._init_settings(); @@ -1220,7 +1227,8 @@ export class Actions< if ( this.is_public || !this.store.get("is_loaded") || - this._syncstring == null + this._syncstring == null || + this._syncstring.get_state() != "ready" ) { return; } @@ -2249,7 +2257,7 @@ export class Actions< } this.setFormatError(""); } catch (err) { - this.setFormatError(`${err}`, this._syncstring.to_str()); + this.setFormatError(`${err}`, this._syncstring?.to_str()); } finally { this.set_status(""); } diff --git a/src/packages/frontend/frame-editors/latex-editor/util.ts b/src/packages/frontend/frame-editors/latex-editor/util.ts index 9d8a07e9ef..726b374c3c 100644 --- a/src/packages/frontend/frame-editors/latex-editor/util.ts +++ b/src/packages/frontend/frame-editors/latex-editor/util.ts @@ -15,6 +15,7 @@ import { ExecOptsBlocking } from "@cocalc/util/db-schema/projects"; import { separate_file_extension } from "@cocalc/util/misc"; import { ExecuteCodeOutputAsync } from "@cocalc/util/types/execute-code"; import { TIMEOUT_LATEX_JOB_S } from "./constants"; +import { delay } from "awaiting"; export function pdf_path(path: string): string { // if it is already a pdf, don't change the upper/lower casing -- #4562 @@ -76,7 +77,7 @@ async function gatherJobInfo( set_job_info: (info: ExecuteCodeOutputAsync) => void, path: string, ): Promise { - await new Promise((done) => setTimeout(done, 100)); + await delay(100); let wait_s = 1; try { while (true) { @@ -97,7 +98,7 @@ async function gatherJobInfo( } else { return; } - await new Promise((done) => setTimeout(done, 1000 * wait_s)); + await delay(1000 * wait_s); wait_s = Math.min(5, wait_s + 1); } } catch { @@ -178,7 +179,7 @@ export async function runJob(opts: RunJobOpts): Promise { } catch (err) { if (IS_TIMEOUT_CALLING_PROJECT(err)) { // This will eventually be fine, hopefully. We continue trying to get a reply. - await new Promise((done) => setTimeout(done, 100)); + await delay(100); } else { throw new Error( "Unable to run the compilation. Please check up on the project.", diff --git a/src/packages/frontend/frame-editors/lean-editor/_lean.sass b/src/packages/frontend/frame-editors/lean-editor/_lean.sass deleted file mode 100644 index a693c26733..0000000000 --- a/src/packages/frontend/frame-editors/lean-editor/_lean.sass +++ /dev/null @@ -1,11 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -.Codemirror-lean-messages - padding : 0 2px 0 4px - min-width : 15px - text-align : right - white-space : nowrap - diff --git a/src/packages/frontend/frame-editors/lean-editor/actions.ts b/src/packages/frontend/frame-editors/lean-editor/actions.ts deleted file mode 100644 index 858cb260d7..0000000000 --- a/src/packages/frontend/frame-editors/lean-editor/actions.ts +++ /dev/null @@ -1,370 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -/* -Lean Editor Actions -*/ - -// This should be longer than the time between keystrokes, but -// small enough that it feels fast/responsive. -const DEBOUNCE_MS = 750; - -import { debounce } from "underscore"; -import { List } from "immutable"; -import { Store } from "../../app-framework"; -import { - Actions as BaseActions, - CodeEditorState, -} from "../code-editor/actions"; -import { FrameTree } from "../frame-tree/types"; -import { project_api } from "../generic/client"; -import { capitalize, close } from "@cocalc/util/misc"; -import type { Channel } from "@cocalc/comm/websocket/types"; -import { Task, Message, Completion } from "./types"; -import { update_gutters } from "./gutters"; - -interface LeanEditorState extends CodeEditorState { - info: any; - messages: Message[]; - tasks: Task[]; - sync: { hash: number; time: number }; // hash is the hash of last version sync'd to lean, and time is *when* - syncstring_hash: number; // hash of actual syncstring in client -} - -export class Actions extends BaseActions { - private channel: Channel; - public store: Store; - private gutter_last: { synced: boolean; messages: any; tasks: any }; - private debounced_process_data_queue: Function; - private debounced_update_info: Function; - private debounced_update_gutters: Function; - private debounced_update_status_bar: Function; - private data_queue: any[]; - - async _init2(): Promise { - this.data_queue = []; - - this.debounced_process_data_queue = debounce(() => { - if (this._state === "closed") return; - this.process_data_queue(); - }, DEBOUNCE_MS); - - this.debounced_update_info = debounce(() => { - if (this._state === "closed") return; - this.update_info(); - }, DEBOUNCE_MS); - - this.debounced_update_gutters = debounce(() => { - if (this._state === "closed") return; - this.update_gutters(); - }, DEBOUNCE_MS); - - this.debounced_update_status_bar = debounce(() => { - if (this._state === "closed") return; - this.update_status_bar(); - }, DEBOUNCE_MS); - - this.setState({ - messages: [], - tasks: [], - sync: { hash: 0, time: 0 }, - syncstring_hash: 0, - info: {}, - }); - this.gutter_last = { synced: false, messages: List(), tasks: List() }; - if (!this.is_public) { - this._syncstring.on("change", () => { - this.setState({ - syncstring_hash: this._syncstring.hash_of_live_version(), - }); - this.debounced_update_gutters(); - this.debounced_update_status_bar(); - this.debounced_update_info(); - }); - try { - await this._init_channel(); - } catch (err) { - this.set_error( - // TODO: should retry instead (?) - err + - " -- you might need to refresh your browser or close and open this file." - ); - } - } else { - this._init_value(); - } - } - - async _init_channel(): Promise { - if (this._state === "closed") return; - const api = await project_api(this.project_id); - this.channel = await api.lean_channel(this.path); - const channel: any = this.channel; - channel.on("close", () => { - channel.removeAllListeners(); - channel.conn.once("open", async () => { - await this._init_channel(); - }); - }); - channel.on("data", (x) => { - if (typeof x === "object") { - this.handle_data_from_channel(x); - } - }); - } - - handle_data_from_channel(x: object): void { - if (this._state === "closed") return; - this.data_queue.push(x); - this.debounced_process_data_queue(); - } - - process_data_queue(): void { - // Can easily happen when closing, due to debounce. - if (this._state === "closed") return; - if (this.data_queue.length === 0) { - return; - } - for (const x of this.data_queue) { - if (x.messages !== undefined) { - this.setState({ messages: x.messages }); - } - if (x.tasks !== undefined) { - this.setState({ tasks: x.tasks }); - } - if (x.sync !== undefined) { - this.setState({ sync: x.sync }); - } - } - this.data_queue = []; - this.update_gutters(); - this.update_status_bar(); - } - - async restart(): Promise { - this.set_status("Restarting LEAN ..."); - // Using hash: -1 as a signal for restarting -- yes, that's ugly - this.setState({ - sync: { hash: -1, time: 0 }, - }); - const api = await project_api(this.project_id); - try { - await api.lean({ cmd: "restart" }); - await this.update_info(); - } catch (err) { - this.set_error(`Error restarting LEAN: ${err}`); - } finally { - this.set_status(""); - } - } - - close(): void { - if (this.channel !== undefined) { - try { - this.channel.end(); - } catch (err) { - // pass - } - } - super.close(); - close(this); - this._state = "closed"; // close above clears all attributes, so have to set this afterwards. - } - - update_status_bar = (): void => { - // Can easily happen when closing, due to debounce. - if (this._state === "closed") return; - const synced = - this.store.getIn(["sync", "hash"]) == this.store.get("syncstring_hash"); - const tasks = this.store.unsafe_getIn(["tasks"]); - let status = ""; - if (!synced) { - status += "Syncing... "; - } - if (tasks.size > 0) { - const task = tasks.get(0).toJS(); - status += `${capitalize(task.desc)}. Processing lines ${task.pos_line}-${ - task.end_pos_line - }...`; - } - //console.log("update_status_bar", status); - this.set_status(status); - }; - - update_gutters = (): void => { - // Can easily happen when closing, due to debounce. - if (this._state === "closed") return; - const synced = - this.store.getIn(["sync", "hash"]) == this.store.get("syncstring_hash"); - const messages = this.store.unsafe_getIn(["messages"]); - const tasks = this.store.unsafe_getIn(["tasks"]); - const last = this.gutter_last; - if ( - synced === last.synced && - messages === last.messages && - tasks === last.tasks - ) { - return; - } - this.gutter_last = { synced, messages, tasks }; - this.clear_gutter("Codemirror-lean-messages"); - const cm = this._get_cm(); - if (cm === undefined) { - return; // satisfy typescript - } - update_gutters({ - cm, - synced, - messages, - tasks, - set_gutter: (line, component) => { - this.set_gutter_marker({ - line, - component, - gutter_id: "Codemirror-lean-messages", - }); - }, - }); - }; - - _raw_default_frame_tree(): FrameTree { - if (this.is_public) { - return { type: "cm" }; - } else { - return { - direction: "col", - type: "node", - first: { - type: "cm-lean", - }, - second: { - direction: "row", - type: "node", - first: { - type: "lean-messages", - }, - second: { - direction: "row", - type: "node", - first: { - type: "lean-info", - }, - second: { - type: "lean-help", - }, - }, - }, - }; - } - } - - // uses API to get running version of LEAN server. - // I'm just implementing this now; not needed yet. - async version(): Promise { - const api = await project_api(this.project_id); - return await api.lean({ cmd: "version" }); - } - - // Use the backend LEAN server via the api to complete - // at the given position. - async complete(line: number, column: number): Promise { - if (!(await this.ensure_latest_changes_are_saved())) { - return []; - } - - this.set_status(`Completing at line ${line + 1}...`); - try { - const api = await project_api(this.project_id); - return await api.lean({ - path: this.path, - cmd: "complete", - line: line + 1, // codemirror is 0 based but lean is 1-based. - column, - }); - } catch (err) { - err = err.toString(); - if (err === "timeout" || err === "Error: interrupted") { - // user likely doesn't care about error report if this is the reason. - return []; - } - this.set_error(`Error getting completions on line ${line + 1} -- ${err}`); - return []; - } finally { - this.set_status(""); - } - } - - // Use the backend LEAN server via the api to get info - // at the given position. - async info(line: number, column: number): Promise { - if (!(await this.ensure_latest_changes_are_saved())) { - return; - } - - this.set_status(`Get info about line ${line + 1}...`); - try { - const api = await project_api(this.project_id); - return await api.lean({ - path: this.path, - cmd: "info", - line: line + 1, // codemirror is 0 based but lean is 1-based. - column, - }); - } catch (err) { - err = err.toString(); - if ( - err === "timeout" || - err === "Error: interrupted" || - err === "Error: unknown exception" - ) { - // user likely doesn't care about error report if this is the reason. - return; - } - this.set_error(`Error getting info about line ${line + 1} -- ${err}`); - return; - } finally { - this.set_status(""); - } - } - - async update_info(): Promise { - // Can easily happen when closing, due to debounce. - if (this._state === "closed") return; - const cm = this._recent_cm(); - if (cm == null) { - // e.g., maybe no editor - this.setState({ info: {} }); - return; - } - const cur = cm.getDoc().getCursor(); - if (cur == null) { - this.setState({ info: {} }); - return; - } - const info = await this.info(cur.line, cur.ch); - if (info != null) { - this.setState({ info }); - } - } - - handle_cursor_move(_): void { - this.debounced_update_info(); - } - - public async close_and_halt(_: string): Promise { - this.set_status("Killing LEAN server..."); - const api = await project_api(this.project_id); - try { - await api.lean({ cmd: "kill" }); - } catch (err) { - this.set_error(`Error killing LEAN server: ${err}`); - } finally { - this.set_status(""); - } - // and close this window - const project_actions = this._get_project_actions(); - project_actions.close_tab(this.path); - } -} diff --git a/src/packages/frontend/frame-editors/lean-editor/codemirror-lean-symbols.ts b/src/packages/frontend/frame-editors/lean-editor/codemirror-lean-symbols.ts deleted file mode 100644 index 48d506b7e7..0000000000 --- a/src/packages/frontend/frame-editors/lean-editor/codemirror-lean-symbols.ts +++ /dev/null @@ -1,70 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -/* -Make it so Codemirror has an option to insert LEAN symbols -*/ - -import { substitute_symbols } from "./symbols"; - -import * as CodeMirror from "codemirror"; - -declare module "codemirror" { - function innerMode(mode: any, state: any): any; -} - -CodeMirror.defineOption("leanSymbols", false, function (cm, val, old) { - if (old) { - cm.removeKeyMap("leanSymbols"); - cm.off("mousedown", lean_symbols); - cm.off("blur", lean_symbols); - } - if (!val) { - return; - } - const map = { - name: "leanSymbols", - Enter: lean_symbols, - Space: lean_symbols, - Tab: lean_symbols, - Right: lean_symbols, - Up: lean_symbols, - Down: lean_symbols, - "\\": lean_symbols, - }; - cm.on("mousedown", lean_symbols); - cm.on("blur", lean_symbols); - cm.addKeyMap(map); -}); - -/* -interface Position { - line: number; - ch: number; -} - -interface Selection { - head: Position; - anchor: Position; -} -*/ - -function lean_symbols(cm): any { - if (cm.getOption("disableInput")) { - return CodeMirror.Pass; - } - for (const range of cm.listSelections()) { - const line = range.head.line; - for (const sub of substitute_symbols(cm.getLine(line))) { - const { replacement, from, to } = sub; - cm.replaceRange( - replacement, - { line: line, ch: from }, - { line: line, ch: to } - ); - } - } - return CodeMirror.Pass; -} diff --git a/src/packages/frontend/frame-editors/lean-editor/editor.ts b/src/packages/frontend/frame-editors/lean-editor/editor.ts deleted file mode 100644 index 86b3b64657..0000000000 --- a/src/packages/frontend/frame-editors/lean-editor/editor.ts +++ /dev/null @@ -1,86 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -/* -Top-level react component for editing LEAN documents -*/ - -import { set } from "@cocalc/util/misc"; -import { createEditor } from "../frame-tree/editor"; -import { EditorDescription } from "../frame-tree/types"; -import { LeanCodemirrorEditor } from "./lean-codemirror"; -import { LeanMessages } from "./lean-messages"; -import { LeanInfo } from "./lean-info"; -import { LeanHelp } from "./lean-help"; -import { terminal } from "../terminal-editor/editor"; -import { time_travel } from "../time-travel-editor/editor"; - -const cm_lean: EditorDescription = { - type: "cm-lean", - short: "Input", - name: "Input", - icon: "code", - component: LeanCodemirrorEditor, - commands: set([ - "print", - "decrease_font_size", - "increase_font_size", - "save", - "time_travel", - "replace", - "find", - "goto_line", - "cut", - "paste", - "copy", - "undo", - "redo", - "restart", - "close_and_halt", - ]), - gutters: ["Codemirror-lean-messages"], -} as const; - -const lean_info: EditorDescription = { - type: "lean-info", - short: "Info", - name: "Info at Cursor", // more focused -- usually used in "tactic mode" - icon: "info-circle", - component: LeanInfo as any, // TODO: rclass wrapper does not fit the EditorDescription type - commands: set(["decrease_font_size", "increase_font_size"]), -} as const; - -const lean_messages: EditorDescription = { - type: "lean-messages", - short: "Mesages", - name: "All Messages" /* less focused -- usually used in "term mode" */, - icon: "eye", - component: LeanMessages as any, // TODO: rclass wrapper does not fit the EditorDescription type - commands: set(["decrease_font_size", "increase_font_size"]), -} as const; - -const lean_help: EditorDescription = { - type: "lean-help", - short: "Help", - name: "Help at Cursor", - icon: "question-circle", - component: LeanHelp as any, // TODO: rclass wrapper does not fit the EditorDescription type - commands: set(["decrease_font_size", "increase_font_size"]), -} as const; - -const EDITOR_SPEC = { - "cm-lean": cm_lean, - "lean-info": lean_info, - "lean-messages": lean_messages, - "lean-help": lean_help, - terminal, - time_travel, -} as const; - -export const Editor = createEditor({ - format_bar: false, - editor_spec: EDITOR_SPEC, - display_name: "LeanEditor", -}); diff --git a/src/packages/frontend/frame-editors/lean-editor/gutters.tsx b/src/packages/frontend/frame-editors/lean-editor/gutters.tsx deleted file mode 100644 index c053671ff4..0000000000 --- a/src/packages/frontend/frame-editors/lean-editor/gutters.tsx +++ /dev/null @@ -1,81 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -/* -Manage codemirror gutters that provide messages and other info from the backend LEAN. -*/ - -import { Rendered } from "@cocalc/frontend/app-framework"; -import { List } from "immutable"; -import { Icon, Tip } from "@cocalc/frontend/components"; -import { RenderedMessage, message_color, message_icon } from "./lean-messages"; -import { Message, Task } from "./types"; -import { Editor } from "codemirror"; - -export function update_gutters(opts: { - cm: Editor; - synced: boolean; - set_gutter: Function; - messages: List; - tasks: List; -}): void { - for (const message of opts.messages.toJS()) { - opts.set_gutter( - message.pos_line - 1, - message_component( - message, - opts.synced, - opts.cm.getDoc().getLine(message.pos_line - 1) - ) - ); - } - if (opts.tasks.size > 0) { - let task: Task; - for (task of opts.tasks.toJS()) { - for (let line = task.pos_line; line < task.end_pos_line; line++) { - opts.set_gutter(line - 1, task_component(opts.synced)); - } - } - } -} - -function task_component(synced: boolean): Rendered { - let color; - if (synced) { - color = "#5cb85c"; - } else { - color = "#888"; - } - return ; -} - -function message_component( - message: Message, - synced: boolean, - context: string -): Rendered { - const icon = message_icon(message.severity); - const color = message_color(message.severity, synced); - const content = ; - return ( - {context}} - tip={content} - placement={"right"} - stable={true} - popover_style={{ - padding: 0, - border: `2px solid ${color}`, - borderRadius: "3px", - width: "700px", - maxWidth: "80%", - }} - delayShow={0} - allow_touch={true} - > - - - ); -} diff --git a/src/packages/frontend/frame-editors/lean-editor/lean-codemirror.tsx b/src/packages/frontend/frame-editors/lean-editor/lean-codemirror.tsx deleted file mode 100644 index 77de1ee9e0..0000000000 --- a/src/packages/frontend/frame-editors/lean-editor/lean-codemirror.tsx +++ /dev/null @@ -1,8 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -import { CodemirrorEditor } from "../code-editor/codemirror-editor"; - -export const LeanCodemirrorEditor = CodemirrorEditor; diff --git a/src/packages/frontend/frame-editors/lean-editor/lean-help.tsx b/src/packages/frontend/frame-editors/lean-editor/lean-help.tsx deleted file mode 100644 index d845233fe0..0000000000 --- a/src/packages/frontend/frame-editors/lean-editor/lean-help.tsx +++ /dev/null @@ -1,85 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -import { Map } from "immutable"; -import { is_different } from "@cocalc/util/misc"; -import { Markdown } from "@cocalc/frontend/components"; - -import { Component, Rendered, rclass, rtypes } from "../../app-framework"; - -interface Props { - font_size: number; - // reduxProps: - info: Map; // keys are: doc, source, state, tactic_params, text, type - sync: Map; - syncstring_hash: number; -} - -class LeanHelp extends Component { - static displayName = "LeanHelp"; - - shouldComponentUpdate(next_props): boolean { - return is_different(this.props, next_props, [ - "font_size", - "info", - "sync", - "syncstring_hash", - ]); - } - - static reduxProps({ name }) { - return { - [name]: { - info: rtypes.immutable.Map, - sync: rtypes.immutable.Map, - syncstring_hash: rtypes.number, - }, - }; - } - - render_heading(value: string): Rendered { - return ( -
    - - {value} - -
    - ); - } - - render_doc(): Rendered { - const doc = this.props.info.get("doc"); - const params = this.props.info.get("tactic_params"); - if (!doc && !params) { - return; - } - return ( -
    - {this.render_heading(params)} - -
    - ); - } - - render(): Rendered { - if (this.props.info == null) { - return ; - } - return ( -
    - {this.render_doc()} -
    - ); - } -} - -const LeanHelp0 = rclass(LeanHelp); -export { LeanHelp0 as LeanHelp }; diff --git a/src/packages/frontend/frame-editors/lean-editor/lean-info.tsx b/src/packages/frontend/frame-editors/lean-editor/lean-info.tsx deleted file mode 100644 index 547017cde7..0000000000 --- a/src/packages/frontend/frame-editors/lean-editor/lean-info.tsx +++ /dev/null @@ -1,98 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -import { Map } from "immutable"; -import { is_different } from "@cocalc/util/misc"; -import { Component, Rendered, rclass, rtypes } from "../../app-framework"; - -interface Props { - font_size: number; - // reduxProps: - info: Map; // keys are: doc, source, state, tactic_params, text, type - sync: Map; - syncstring_hash: number; -} - -function render_text(text: string, zoom: number = 100): Rendered { - return ( -
    - {text} -
    - ); -} - -class LeanInfo extends Component { - static displayName = "LeanInfo"; - - shouldComponentUpdate(next_props): boolean { - return is_different(this.props, next_props, [ - "font_size", - "info", - "sync", - "syncstring_hash", - ]); - } - - static reduxProps({ name }) { - return { - [name]: { - info: rtypes.immutable.Map, - sync: rtypes.immutable.Map, - syncstring_hash: rtypes.number, - }, - }; - } - - render_heading(value: string): Rendered { - return ( -
    - - {value} - -
    - ); - } - - render_state(): Rendered { - const state = this.props.info.get("state"); - if (!state) { - return; - } - return ( -
    - {this.render_heading("Tactic State")} - {render_text(state, 105)} -
    - ); - } - - render(): Rendered { - if (this.props.info == null) { - return ; - } - return ( -
    - {this.render_state()} -
    - ); - } -} - -const LeanInfo0 = rclass(LeanInfo); -export { LeanInfo0 as LeanInfo }; diff --git a/src/packages/frontend/frame-editors/lean-editor/lean-messages.tsx b/src/packages/frontend/frame-editors/lean-editor/lean-messages.tsx deleted file mode 100644 index 2fc7b06201..0000000000 --- a/src/packages/frontend/frame-editors/lean-editor/lean-messages.tsx +++ /dev/null @@ -1,275 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -import { List, Map } from "immutable"; -import { Icon, IconName, Gap, TimeAgo } from "@cocalc/frontend/components"; -import { server_time } from "../generic/client"; -import { Message } from "./types"; -import { capitalize, is_different } from "@cocalc/util/misc"; -import { Component, Rendered, rclass, rtypes } from "../../app-framework"; - -interface Props { - font_size: number; - // reduxProps: - messages: List; - tasks: List; - sync: Map; - syncstring_hash: number; -} - -function render_pos(line: number, col: number): Rendered { - return ( - - {line}:{col} - - ); -} - -function render_severity(severity: string): Rendered { - return {severity}; -} - -function render_caption(caption: string): Rendered { - return {caption}; -} - -const COLORS = { - information: "#5bc0de", - error: "#d9534f", - warning: "#f0ad4e", -}; - -export function message_color(severity: string, synced: boolean): string { - if (!synced) { - return "grey"; - } - const color = COLORS[severity]; - return color ? color : "grey"; -} - -const ICONS = { - information: "info-circle", - error: "exclamation-triangle", - warning: "exclamation-circle", -} as { [name: string]: IconName }; - -export function message_icon(severity: string): IconName { - return ICONS[severity] ?? "question-circle"; -} - -function render_text(text: string): Rendered { - return ( -
    - {text} -
    - ); -} - -// nothing extra yet. -interface MessageProps { - message: Message; - synced: boolean; -} - -export class RenderedMessage extends Component { - static displayName = "LeanMessage"; - - render(): Rendered { - const message = this.props.message; - const color = message_color(message.severity, this.props.synced); - return ( -
    -
    - - - {message.pos_line !== undefined && message.pos_col !== undefined - ? render_pos(message.pos_line, message.pos_col) - : undefined} - - {message.severity !== undefined - ? render_severity(message.severity) - : undefined} - - {message.caption !== undefined - ? render_caption(message.caption) - : undefined} -
    - {message.text !== undefined ? render_text(message.text) : undefined} -
    - ); - } -} - -class LeanMessages extends Component { - static displayName = "LeanMessages"; - - shouldComponentUpdate(next_props): boolean { - return is_different(this.props, next_props, [ - "font_size", - "messages", - "tasks", - "sync", - "syncstring_hash", - ]); - } - - static reduxProps({ name }) { - return { - [name]: { - messages: rtypes.immutable.List, - tasks: rtypes.immutable.List, - sync: rtypes.immutable.Map, - syncstring_hash: rtypes.number, - }, - }; - } - - render_message(key, message): Rendered { - return ( -
    - -
    - ); - } - - render_messages(): Rendered | Rendered[] { - if (!this.props.messages) { - return
    (nothing)
    ; - } - const v: Rendered[] = []; - const messages = this.props.messages.toJS(); - messages.sort(cmp_messages); - let i = 0; - for (const message of messages) { - v.push(this.render_message(i, message)); - i += 1; - } - return v; - } - - render_task(i, task): Rendered { - return ( -
    - - - {capitalize(task.desc)} - (Processing lines {task.pos_line}-{task.end_pos_line}) -
    - ); - } - - render_done(): Rendered { - return ( -
    - -
    - ); - } - - render_tasks(): Rendered | Rendered[] { - if (!this.props.tasks || this.props.tasks.size === 0) { - return this.render_done(); - } - const v: Rendered[] = []; - let i = 0; - for (const task of this.props.tasks.toJS()) { - v.push(this.render_task(i, task)); - i += 1; - } - return v; - } - - render_last_run_time(): Rendered { - const time = this.props.sync.get("time"); - if (!time) { - return; - } - const t = - Date.now() - Math.max(0, server_time().valueOf() - time); - return ; - } - - render_sync(): Rendered { - let msg: string | Rendered; - if (this.props.sync.get("hash") === -1) { - msg = `Restarting…`; - } else if (this.props.sync.get("hash") === this.props.syncstring_hash) { - msg = Synced ({this.render_last_run_time()}); - } else { - msg = `Syncing…`; - } - return
    {msg}
    ; - } - - render(): Rendered { - return ( -
    - {this.render_sync()} - {this.render_tasks()} - {this.render_messages()} -
    - ); - } -} - -const LeanMessages0 = rclass(LeanMessages); -export { LeanMessages0 as LeanMessages }; - -function cmp_messages(m0: Message, m1: Message): number { - if ( - m0.pos_line < m1.pos_line || - (m0.pos_line === m1.pos_line && m0.pos_col < m1.pos_col) - ) { - return -1; - } else if (m0.pos_line === m1.pos_line && m0.pos_col === m1.pos_col) { - return 0; - } else { - return 1; - } -} diff --git a/src/packages/frontend/frame-editors/lean-editor/register.ts b/src/packages/frontend/frame-editors/lean-editor/register.ts deleted file mode 100644 index 625d719ae8..0000000000 --- a/src/packages/frontend/frame-editors/lean-editor/register.ts +++ /dev/null @@ -1,23 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -/* -Register the LEAN theorem prover editor -*/ - -import { register_file_editor } from "../frame-tree/register"; -require("./_lean.sass"); - -register_file_editor({ - ext: "lean", - editor: async () => { - // Load plugin so that codemirror can automatically insert LEAN symbols - await import("./codemirror-lean-symbols"); - // Register the tab completion helper for lean mode. - await import("./tab-completion"); - return await import("./editor"); - }, - actions: async () => await import("./actions"), -}); diff --git a/src/packages/frontend/frame-editors/lean-editor/symbols.ts b/src/packages/frontend/frame-editors/lean-editor/symbols.ts deleted file mode 100644 index a4a0817802..0000000000 --- a/src/packages/frontend/frame-editors/lean-editor/symbols.ts +++ /dev/null @@ -1,2307 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -interface Sub { - from: number; - to: number; - replacement: string; -} - -export function substitute_symbols(value: string, j: number = 0): Sub[] { - const i = value.indexOf("\\", j); - if (i === -1) { - return []; - } - // Replace the one at i. - const s = value.slice(i + 1); - if (s[0] === undefined || /\s/.test(s[0])) { - return substitute_symbols(value, i + 1); - } - let closest: string = ""; - for (const m in TRANSLATIONS) { - if (s.slice(0, m.length) == m && m.length > closest.length) { - closest = m; - } - } - if (closest === "") { - return substitute_symbols(value, i + 1); - } - const replacement = TRANSLATIONS[closest]; - value = value.slice(0, i) + replacement + value.slice(i + closest.length + 1); - const from = i, - to = i + closest.length + 1; - const result: Sub[] = [{ replacement, from, to }]; - for (const x of substitute_symbols(value, i + 1)) { - result.push(x); - } - return result; -} - -/* This is https://github.com/leanprover/vscode-lean/blob/master/translations.json - */ -const TRANSLATIONS = { - note: "♩", - notin: "∉", - not: "¬", - nomisma: "𐆎", - nin: "∌", - ni: "∋", - nattrans: "⟹", - nat_trans: "⟹", - natural: "♮", - nat: "ℕ", - naira: "₦", - nabla: "∇", - napprox: "≉", - numero: "№", - nu: "ν", - nLeftarrow: "⇍", - nLeftrightarrow: "⇎", - nRightarrow: "⇏", - nVDash: "⊯", - nVdash: "⊮", - ncong: "≇", - nearrow: "↗", - neg: "¬", - nequiv: "≢", - neq: "≠", - nexists: "∄", - ne: "≠", - ngeqq: "≱", - ngeqslant: "≱", - ngeq: "≱", - ngtr: "≯", - nleftarrow: "↚", - nleftrightarrow: "↮", - nleqq: "≰", - nleqslant: "≰", - nleq: "≰", - nless: "≮", - nmid: "∤", - nparallel: "∦", - npreceq: "⋠", - nprec: "⊀", - nrightarrow: "↛", - nshortmid: "∤", - nshortparallel: "∦", - nsimeq: "≄", - nsim: "≁", - nsubseteqq: "⊈", - nsubseteq: "⊈", - nsubset: "⊄", - nsucceq: "⋡", - nsucc: "⊁", - nsupseteqq: "⊉", - nsupseteq: "⊉", - nsupset: "⊅", - ntrianglelefteq: "⋬", - ntriangleleft: "⋪", - ntrianglerighteq: "⋭", - ntriangleright: "⋫", - nvDash: "⊭", - nvdash: "⊬", - nwarrow: "↖", - eqn: "≠", - equiv: "≃", - eqcirc: "≖", - eqcolon: "≕", - eqslantgtr: "⋝", - eqslantless: "⋜", - eq: "=", - entails: "⊢", - en: "–", - exn: "∄", - exists: "∃", - ex: "∃", - emptyset: "∅", - empty: "∅", - em: "—", - epsilon: "ε", - eps: "ε", - euro: "€", - eta: "η", - ell: "ℓ", - iso: "≅", - inn: "∉", - intersection: "∩", - intercal: "⊺", - integral: "∫", - int: "ℤ", - inv: "⁻¹", - increment: "∆", - infty: "∞", - inf: "∞", - in: "∈", - iff: "↔", - imp: "→", - imath: "ı", - iota: "ι", - i: "∩", - "=n": "≠", - "==n": "≢", - "===": "≣", - "==>": "⟹", - "==": "≡", - "=:": "≕", - "=o": "≗", - "=>n": "⇏", - "=>": "⇒", - '=\\"{u}': "ṻ", - '=\\"{U}': "Ṻ", - '=\\"u': "ṻ", - '=\\"U': "Ṻ", - "=\\'{o}": "ṓ", - "=\\'{O}": "Ṓ", - "=\\'{e}": "ḗ", - "=\\'{E}": "Ḗ", - "=\\'o": "ṓ", - "=\\'O": "Ṓ", - "=\\'e": "ḗ", - "=\\'E": "Ḗ", - "=\\`{o}": "ṑ", - "=\\`{O}": "Ṑ", - "=\\`{e}": "ḕ", - "=\\`{E}": "Ḕ", - "=\\`o": "ṑ", - "=\\`O": "Ṑ", - "=\\`e": "ḕ", - "=\\`E": "Ḕ", - "=\\AE": "Ǣ", - "=\\ae": "ǣ", - "={g}": "ḡ", - "={G}": "Ḡ", - "={y}": "ȳ", - "={Y}": "Ȳ", - "={u}": "ū", - "={U}": "Ū", - "={o}": "ō", - "={O}": "Ō", - "={i}": "ī", - "={I}": "Ī", - "={e}": "ē", - "={E}": "Ē", - "={a}": "ā", - "={A}": "Ā", - "={}": "¯", - "={\\i}": "ī", - "={\\AE}": "Ǣ", - "={\\ae}": "ǣ", - "=g": "ḡ", - "=G": "Ḡ", - "=y": "ȳ", - "=Y": "Ȳ", - "=u": "ū", - "=U": "Ū", - "=O": "Ō", - "=i": "ī", - "=I": "Ī", - "=e": "ē", - "=E": "Ē", - "=a": "ā", - "=A": "Ā", - "=": "̄", - "~n": "≁", - "~~n": "≉", - "~~~": "≋", - "~~-": "≊", - "~~": "≈", - "~-n": "≄", - "~-": "≃", - "~=n": "≇", - "~=": "≅", - "~{y}": "ỹ", - "~{Y}": "Ỹ", - "~{e}": "ẽ", - "~{E}": "Ẽ", - "~{v}": "ṽ", - "~{V}": "Ṽ", - "~{u}": "ũ", - "~{U}": "Ũ", - "~{i}": "ĩ", - "~{I}": "Ĩ", - "~{o}": "õ", - "~{n}": "ñ", - "~{a}": "ã", - "~{O}": "Õ", - "~{N}": "Ñ", - "~{A}": "Ã", - "~{}": "˜", - "~{\\i}": "ĩ", - "~y": "ỹ", - "~Y": "Ỹ", - "~e": "ẽ", - "~E": "Ẽ", - "~v": "ṽ", - "~V": "Ṽ", - "~\\'{u}": "ṹ", - "~\\'{U}": "Ṹ", - "~\\'{o}": "ṍ", - "~\\'{O}": "Ṍ", - "~\\'u": "ṹ", - "~\\'U": "Ṹ", - "~\\'o": "ṍ", - "~\\'O": "Ṍ", - '~\\"{o}': "ṏ", - '~\\"{O}': "Ṏ", - '~\\"o': "ṏ", - '~\\"O': "Ṏ", - "~\\={o}": "ȭ", - "~\\={O}": "Ȭ", - "~\\=o": "ȭ", - "~\\=O": "Ȭ", - "~u": "ũ", - "~U": "Ũ", - "~i": "ĩ", - "~I": "Ĩ", - "~o": "õ", - "~a": "ã", - "~O": "Õ", - "~N": "Ñ", - "~A": "Ã", - "~": "∼", - homotopy: "∼", - hom: "⟶", - hori: "ϩ", - hookleftarrow: "↩", - hookrightarrow: "↪", - hryvnia: "₴", - heta: "ͱ", - heartsuit: "♥", - hbar: "ℏ", - ":~": "∻", - ":=": "≔", - "::-": "∺", - "::": "∷", - ":": "∶", - "-~": "≂", - "-|": "⊣", - "-1e": "⁻¹ᵉ", - "-1f": "⁻¹ᶠ", - "-1g": "⁻¹ᵍ", - "-1h": "⁻¹ʰ", - "-1i": "⁻¹ⁱ", - "-1m": "⁻¹ᵐ", - "-1o": "⁻¹ᵒ", - "-1r": "⁻¹ʳ", - "-1p": "⁻¹ᵖ", - "-1s": "⁻¹ˢ", - "-1v": "⁻¹ᵛ", - "-1E": "⁻¹ᴱ", - "-1": "⁻¹", - "-2o": "⁻²ᵒ", - "-2": "⁻²", - "-3": "⁻³", - "-:": "∹", - "->n": "↛", - "->": "→", - "-->": "⟶", - "---": "─", - "--=": "═", - "--_": "━", - "--.": "╌", - "-o": "⊸", - "-": "­", - ".=.": "≑", - ".=": "≐", - ".+": "∔", - ".-": "∸", - "...": "⋯", - ".{y}": "ẏ", - ".{Y}": "Ẏ", - ".{x}": "ẋ", - ".{X}": "Ẋ", - ".{w}": "ẇ", - ".{W}": "Ẇ", - ".{t}": "ṫ", - ".{T}": "Ṫ", - ".{s}": "ṡ", - ".{S}": "Ṡ", - ".{r}": "ṙ", - ".{R}": "Ṙ", - ".{p}": "ṗ", - ".{P}": "Ṗ", - ".{n}": "ṅ", - ".{N}": "Ṅ", - ".{m}": "ṁ", - ".{M}": "Ṁ", - ".{h}": "ḣ", - ".{H}": "Ḣ", - ".{f}": "ḟ", - ".{F}": "Ḟ", - ".{d}": "ḋ", - ".{D}": "Ḋ", - ".{b}": "ḃ", - ".{B}": "Ḃ", - ".{o}": "ȯ", - ".{O}": "Ȯ", - ".{a}": "ȧ", - ".{A}": "Ȧ", - ".{z}": "ż", - ".{Z}": "Ż", - ".{I}": "İ", - ".{g}": "ġ", - ".{G}": "Ġ", - ".{e}": "ė", - ".{E}": "Ė", - ".{c}": "ċ", - ".{C}": "Ċ", - ".{}": "˙", - ".y": "ẏ", - ".Y": "Ẏ", - ".x": "ẋ", - ".X": "Ẋ", - ".w": "ẇ", - ".W": "Ẇ", - ".t": "ṫ", - ".T": "Ṫ", - ".s": "ṡ", - ".S": "Ṡ", - ".r": "ṙ", - ".R": "Ṙ", - ".p": "ṗ", - ".P": "Ṗ", - ".n": "ṅ", - ".N": "Ṅ", - ".m": "ṁ", - ".M": "Ṁ", - ".h": "ḣ", - ".H": "Ḣ", - ".f": "ḟ", - ".F": "Ḟ", - ".d": "ḋ", - ".D": "Ḋ", - ".b": "ḃ", - ".B": "Ḃ", - ".\\={o}": "ȱ", - ".\\={O}": "Ȱ", - ".\\={a}": "ǡ", - ".\\={A}": "Ǡ", - ".\\=o": "ȱ", - ".\\=O": "Ȱ", - ".\\=a": "ǡ", - ".\\=A": "Ǡ", - ".o": "ȯ", - ".O": "Ȯ", - ".a": "ȧ", - ".A": "Ȧ", - ".z": "ż", - ".Z": "Ż", - ".I": "İ", - ".g": "ġ", - ".G": "Ġ", - ".e": "ė", - ".E": "Ė", - ".c": "ċ", - ".C": "Ċ", - ".": "∙", - "(=": "≘", - "(b)": "⒝", - "(b": "⟅", - "(0)": "⓪", - "(1)": "⑴", - "(10)": "⑽", - "(11)": "⑾", - "(12)": "⑿", - "(13)": "⒀", - "(14)": "⒁", - "(15)": "⒂", - "(16)": "⒃", - "(17)": "⒄", - "(18)": "⒅", - "(19)": "⒆", - "(2)": "⑵", - "(20)": "⒇", - "(3)": "⑶", - "(4)": "⑷", - "(5)": "⑸", - "(6)": "⑹", - "(7)": "⑺", - "(8)": "⑻", - "(9)": "⑼", - "(a)": "⒜", - "(c)": "⒞", - "(d)": "⒟", - "(e)": "⒠", - "(f)": "⒡", - "(g)": "⒢", - "(h)": "⒣", - "(i)": "⒤", - "(j)": "⒥", - "(k)": "⒦", - "(l)": "⒧", - "(m)": "⒨", - "(n)": "⒩", - "(o)": "⒪", - "(p)": "⒫", - "(q)": "⒬", - "(r)": "⒭", - "(s)": "⒮", - "(t)": "⒯", - "(u)": "⒰", - "(v)": "⒱", - "(w)": "⒲", - "(x)": "⒳", - "(y)": "⒴", - "(z)": "⒵", - "(": "(", - "and=": "≙", - and: "∧", - angle: "∟", - angstrom: "Å", - all: "∀", - alpha: "α", - aleph: "ℵ", - asterisk: "⁎", - ast: "∗", - asymp: "≍", - apl: "⌶", - approxeq: "≊", - approx: "≈", - aa: "å", - ae: "æ", - austral: "₳", - afghani: "؋", - amalg: "∐", - a: "α", - "or=": "≚", - ordfeminine: "ª", - ordmasculine: "º", - or: "∨", - opf: "ᵒᵖᶠ", - oplus: "⊕", - op: "ᵒᵖ", - "o+": "⊕", - "o--": "⊖", - "o-": "⊝", - ox: "⊗", - "o/": "⊘", - "o.": "⊙", - oo: "⊚", - "o*": "∘*", - "o=": "⊜", - oe: "œ", - octagonal: "🛑", - ohm: "Ω", - ounce: "℥", - omega: "ω", - omicron: "ο", - ominus: "⊖", - odot: "⊙", - oint: "∮", - oslash: "⊘", - otimes: "⊗", - o: "∘", - "*=": "≛", - "*": "⋆", - "t=": "≜", - transport: "▹", - trans: "▹", - triangledown: "▿", - trianglelefteq: "⊴", - triangleleft: "◃", - triangleq: "≜", - trianglerighteq: "⊵", - triangleright: "▹", - triangle: "▵", - tr: "⬝", - tb: "◂", - twoheadleftarrow: "↞", - twoheadrightarrow: "↠", - tw: "◃", - tie: "⁀", - times: "×", - theta: "θ", - therefore: "∴", - thickapprox: "≈", - thicksim: "∼", - th: "θ", - telephone: "℡", - tenge: "₸", - textmusicalnote: "♪", - textmu: "µ", - textfractionsolidus: "⁄", - textbigcircle: "⃝", - textbaht: "฿", - textdied: "✝", - textdiscount: "⁒", - textcolonmonetary: "₡", - textcircledP: "℗", - textwon: "₩", - textnaira: "₦", - textnumero: "№", - textpeso: "₱", - textpertenthousand: "‱", - textlira: "₤", - textlquill: "⁅", - textrecipe: "℞", - textreferencemark: "※", - textrquill: "⁆", - textinterrobang: "‽", - textestimated: "℮", - textopenbullet: "◦", - tugrik: "₮", - tau: "τ", - top: "⊤", - to: "→", - t: "▸", - "def=": "≝", - defs: "≙", - degree: "°", - dei: "ϯ", - delta: "δ", - doteqdot: "≑", - doteq: "≐", - dotplus: "∔", - dotsquare: "⊡", - dot: "⬝", - dong: "₫", - downarrow: "↓", - downdownarrows: "⇊", - downleftharpoon: "⇃", - downrightharpoon: "⇂", - "dr-": "↘", - "dr=": "⇘", - drachma: "₯", - dr: "↘", - "dl-": "↙", - "dl=": "⇙", - dl: "↙", - "d-2": "⇊", - "d-u-": "⇵", - "d-|": "↧", - "d-": "↓", - "d==": "⟱", - "d=": "⇓", - "dd-": "↡", - ddagger: "‡", - ddag: "‡", - ddots: "⋱", - dz: "↯", - dib: "◆", - diw: "◇", - "di.": "◈", - die: "⚀", - division: "÷", - divideontimes: "⋇", - div: "÷", - diameter: "⌀", - diamondsuit: "♢", - diamond: "⋄", - digamma: "ϝ", - di: "◆", - "d{y}": "ỵ", - "d{Y}": "Ỵ", - "d{u}": "ụ", - "d{U}": "Ụ", - "d{o}": "ọ", - "d{O}": "Ọ", - "d{i}": "ị", - "d{I}": "Ị", - "d{e}": "ẹ", - "d{E}": "Ẹ", - "d{a}": "ạ", - "d{A}": "Ạ", - "d{z}": "ẓ", - "d{Z}": "Ẓ", - "d{w}": "ẉ", - "d{W}": "Ẉ", - "d{v}": "ṿ", - "d{V}": "Ṿ", - "d{t}": "ṭ", - "d{T}": "Ṭ", - "d{s}": "ṣ", - "d{S}": "Ṣ", - "d{r}": "ṛ", - "d{R}": "Ṛ", - "d{n}": "ṇ", - "d{N}": "Ṇ", - "d{m}": "ṃ", - "d{M}": "Ṃ", - "d{l}": "ḷ", - "d{L}": "Ḷ", - "d{k}": "ḳ", - "d{K}": "Ḳ", - "d{h}": "ḥ", - "d{H}": "Ḥ", - "d{d}": "ḍ", - "d{D}": "Ḍ", - "d{b}": "ḅ", - "d{B}": "Ḅ", - "d\\.{s}": "ṩ", - "d\\.{S}": "Ṩ", - "d\\.s": "ṩ", - "d\\.S": "Ṩ", - "d\\={r}": "ṝ", - "d\\={R}": "Ṝ", - "d\\={l}": "ḹ", - "d\\={L}": "Ḹ", - "d\\=r": "ṝ", - "d\\=R": "Ṝ", - "d\\=l": "ḹ", - "d\\=L": "Ḹ", - dagger: "†", - dag: "†", - daleth: "ℸ", - dashv: "⊣", - dh: "ð", - d: "↓", - "m=": "≞", - member: "∈", - mem: "∈", - measuredangle: "∡", - male: "♂", - maltese: "✠", - manat: "₼", - mapsto: "↦", - "mathscr{I}": "ℐ", - minus: "−", - mill: "₥", - micro: "µ", - mid: "∣", - multiplication: "×", - multimap: "⊸", - mu: "μ", - mho: "℧", - models: "⊧", - mp: "∓", - "?=": "≟", - "??": "⁇", - "?!": "‽", - "?": "¿", - prohibited: "🛇", - prod: "∏", - propto: "∝", - precapprox: "≾", - preceq: "≼", - precnapprox: "⋨", - precnsim: "⋨", - precsim: "≾", - prec: "≺", - prime: "′", - pr: "↣", - powerset: "𝒫", - pounds: "£", - pound: "£", - pab: "▰", - paw: "▱", - partnership: "㉐", - partial: "∂", - paragraph: "¶", - parallel: "∥", - pa: "▰", - pm: "±", - perp: "⊥", - permil: "‰", - per: "⅌", - peso: "₱", - peseta: "₧", - pilcrow: "¶", - pitchfork: "⋔", - pi: "π", - psi: "ψ", - phi: "φ", - "1": "₁", - "2": "₂", - "3": "₃", - "4": "₄", - "5": "₅", - "6": "₆", - "7": "₇", - "8<": "✂", - "8": "₈", - "9": "₉", - "0": "₀", - leqn: "≰", - leqq: "≦", - leqslant: "≤", - leq: "≤", - len: "≰", - leadsto: "↝", - leftarrowtail: "↢", - leftarrow: "←", - leftharpoondown: "↽", - leftharpoonup: "↼", - leftleftarrows: "⇇", - leftrightarrows: "⇆", - leftrightarrow: "↔", - leftrightharpoons: "⇋", - leftrightsquigarrow: "↭", - leftthreetimes: "⋋", - lessapprox: "≲", - lessdot: "⋖", - lesseqgtr: "⋚", - lesseqqgtr: "⋚", - lessgtr: "≶", - lesssim: "≲", - le: "≤", - lub: "⊔", - "lr--": "⟷", - "lr-n": "↮", - "lr-": "↔", - "lr=n": "⇎", - "lr=": "⇔", - "lr~": "↭", - lrcorner: "⌟", - lr: "↔", - "l-2": "⇇", - "l-r-": "⇆", - "l--": "⟵", - "l-n": "↚", - "l-|": "↤", - "l->": "↢", - "l-": "←", - "l==": "⇚", - "l=n": "⇍", - "l=": "⇐", - "l~": "↜", - "ll-": "↞", - llcorner: "⌞", - llbracket: "〚", - ll: "≪", - lbag: "⟅", - lambdabar: "ƛ", - lambda: "λ", - lamda: "λ", - lam: "λ", - lari: "₾", - langle: "⟨", - lira: "₤", - lceil: "⌈", - ldots: "…", - ldq: "“", - ldata: "《", - lfloor: "⌊", - lhd: "◁", - lnapprox: "⋦", - lneqq: "≨", - lneq: "≨", - lnsim: "⋦", - lnot: "¬", - longleftarrow: "←", - longleftrightarrow: "↔", - longmapsto: "↦", - longrightarrow: "→", - looparrowleft: "↫", - looparrowright: "↬", - lozenge: "✧", - lq: "‘", - ltimes: "⋉", - lvertneqq: "≨", - l: "←", - geqn: "≱", - geqq: "≧", - geqslant: "≥", - geq: "≥", - gen: "≱", - gets: "←", - ge: "≥", - glb: "⊓", - glqq: "„", - glq: "‚", - guarani: "₲", - gangia: "ϫ", - gamma: "γ", - ggg: "⋙", - gg: "≫", - gimel: "ℷ", - gnapprox: "⋧", - gneqq: "≩", - gneq: "≩", - gnsim: "⋧", - gtrapprox: "≳", - gtrdot: "⋗", - gtreqless: "⋛", - gtreqqless: "⋛", - gtrless: "≷", - gtrsim: "≳", - gvertneqq: "≩", - grqq: "“", - grq: "‘", - g: "γ", - "<=n": "≰", - "<=>n": "⇎", - "<=>": "⇔", - "<=": "≤", - "n": "↮", - "<->": "↔", - "<-->": "⟷", - "<--": "⟵", - "<-n": "↚", - "<-": "←", - "<<": "⟪", - "<": "⟨", - ">=n": "≱", - ">=": "≥", - ">n": "≯", - ">~nn": "≵", - ">~n": "⋧", - ">~": "≳", - ">>": "⟫", - ">": "⟩", - ssubn: "⊄", - ssub: "⊂", - ssupn: "⊅", - ssup: "⊃", - ssqub: "⊏", - ssqup: "⊐", - ss: "ß", - subn: "⊈", - subseteqq: "⊆", - subseteq: "⊆", - subsetneqq: "⊊", - subsetneq: "⊊", - subset: "⊂", - sub: "⊆", - supn: "⊉", - supseteqq: "⊇", - supseteq: "⊇", - supsetneqq: "⊋", - supsetneq: "⊋", - supset: "⊃", - sup: "⊇", - surd3: "∛", - surd4: "∜", - surd: "√", - succapprox: "≿", - succcurlyeq: "≽", - succeq: "≽", - succnapprox: "⋩", - succnsim: "⋩", - succsim: "≿", - succ: "≻", - sum: "∑", - squbn: "⋢", - squb: "⊑", - squpn: "⋣", - squp: "⊒", - square: "□", - squigarrowright: "⇝", - sqb: "■", - sqw: "□", - "sq.": "▣", - sqo: "▢", - sqcap: "⊓", - sqcup: "⊔", - sqsubseteq: "⊑", - sqsubset: "⊏", - sqsupseteq: "⊒", - sqsupset: "⊐", - sq: "◾", - sy: "⁻¹", - st4: "✦", - st6: "✶", - st8: "✴", - st12: "✹", - stigma: "ϛ", - star: "⋆", - straightphi: "φ", - st: "⋆", - spesmilo: "₷", - spadesuit: "♠", - sphericalangle: "∢", - section: "§", - searrow: "↘", - setminus: "∖", - san: "ϻ", - sampi: "ϡ", - shortmid: "∣", - shortparallel: "∥", - sho: "ϸ", - shima: "ϭ", - shei: "ϣ", - sharp: "♯", - sigma: "σ", - simeq: "≃", - sim: "∼", - sbs: "﹨", - smallamalg: "∐", - smallsetminus: "∖", - smallsmile: "⌣", - smile: "⌣", - swarrow: "↙", - T1: "Type₁", - T2: "Type₂", - "T+": "Type₊", - Tr: "◀", - Tb: "◀", - Tw: "◁", - Tau: "Τ", - Theta: "Θ", - TH: "Þ", - union: "∪", - undertie: "‿", - uncertainty: "⯑", - un: "∪", - "u+": "⊎", - "u.": "⊍", - "ud-|": "↨", - "ud-": "↕", - "ud=": "⇕", - ud: "↕", - "ul-": "↖", - "ul=": "⇖", - ulcorner: "⌜", - ul: "↖", - "ur-": "↗", - "ur=": "⇗", - urcorner: "⌝", - ur: "↗", - "u-2": "⇈", - "u-d-": "⇅", - "u-|": "↥", - "u-": "↑", - "u==": "⟰", - "u=": "⇑", - "uu-": "↟", - uu: "ŭ", - "u\\d{a}": "ặ", - "u\\d{A}": "Ặ", - "u\\~{a}": "ẵ", - "u\\~{A}": "Ẵ", - "u\\~a": "ẵ", - "u\\~A": "Ẵ", - "u\\`{a}": "ằ", - "u\\`{A}": "Ằ", - "u\\`a": "ằ", - "u\\`A": "Ằ", - "u\\'{a}": "ắ", - "u\\'{A}": "Ắ", - "u\\'a": "ắ", - "u\\'A": "Ắ", - "u{u}": "ŭ", - "u{U}": "Ŭ", - "u{o}": "ŏ", - "u{O}": "Ŏ", - "u{I}": "Ĭ", - "u{g}": "ğ", - "u{G}": "Ğ", - "u{e}": "ĕ", - "u{E}": "Ĕ", - "u{a}": "ă", - "u{A}": "Ă", - "u{}": "˘", - "u{\\i}": "ĭ", - "u{i}": "ĭ", - uU: "Ŭ", - uo: "ŏ", - uO: "Ŏ", - ui: "ĭ", - uI: "Ĭ", - ug: "ğ", - uG: "Ğ", - ue: "ĕ", - uE: "Ĕ", - ua: "ă", - uA: "Ă", - upsilon: "υ", - uparrow: "↑", - updownarrow: "↕", - upleftharpoon: "↿", - uplus: "⊎", - uprightharpoon: "↾", - upuparrows: "⇈", - u: "↑", - And: "⋀", - AA: "Å", - AE: "Æ", - Alpha: "Α", - A: "𝔸", - Or: "⋁", - "O+": "⨁", - Ox: "⨂", - "O.": "⨀", - "O*": "⍟", - OE: "Œ", - Omega: "Ω", - Omicron: "Ο", - O: "Ø", - Int: "ℤ", - Iota: "Ι", - Im: "ℑ", - I: "⋂", - Un: "⋃", - "U+": "⨄", - "U.": "⨃", - "U{o}": "ő", - Uo: "ő", - Upsilon: "Υ", - Uparrow: "⇑", - Updownarrow: "⇕", - Glb: "⨅", - "Gl-": "ƛ", - Gl: "λ", - Gangia: "Ϫ", - Gamma: "Γ", - Ga: "α", - GA: "Α", - Gb: "β", - GB: "Β", - Gg: "γ", - GG: "Γ", - Gd: "δ", - GD: "Δ", - Ge: "ε", - GE: "Ε", - Gz: "ζ", - GZ: "Ζ", - Gth: "θ", - Gt: "τ", - GTH: "Θ", - GT: "Τ", - Gi: "ι", - GI: "Ι", - Gk: "κ", - GK: "Κ", - GL: "Λ", - Gm: "μ", - GM: "Μ", - Gn: "ν", - GN: "Ν", - Gx: "ξ", - GX: "Ξ", - Gr: "ρ", - GR: "Ρ", - Gs: "σ", - GS: "Σ", - Gu: "υ", - GU: "Υ", - Gf: "φ", - GF: "Φ", - Gc: "χ", - GC: "Χ", - Gp: "ψ", - GP: "Ψ", - Go: "ω", - GO: "Ω", - Lub: "⨆", - Lambda: "Λ", - Lamda: "Λ", - Leftarrow: "⇐", - Leftrightarrow: "⇔", - Letter: "✉", - Lleftarrow: "⇚", - Ll: "⋘", - Longleftarrow: "⇐", - Longleftrightarrow: "⇔", - Longrightarrow: "⇒", - Lsh: "↰", - L: "Ł", - "|-n": "⊬", - "|-": "⊢", - "|=n": "⊭", - "|=": "⊨", - "||-n": "⊮", - "||-": "⊩", - "||=n": "⊯", - "||=": "⊫", - "|||-": "⊪", - "||n": "∦", - "||": "∥", - "|n": "∤", - "|": "∣", - Com: "ℂ", - Chi: "Χ", - Cap: "⋒", - Cup: "⋓", - C: "∁", - cul: "⌜", - cuL: "⌈", - currency: "¤", - curlyeqprec: "⋞", - curlyeqsucc: "⋟", - curlypreceq: "≼", - curlyvee: "⋎", - curlywedge: "⋏", - curvearrowleft: "↶", - curvearrowright: "↷", - cur: "⌝", - cuR: "⌉", - cup: "∪", - cu: "⌜", - cll: "⌞", - clL: "⌊", - clr: "⌟", - clR: "⌋", - clubsuit: "♣", - cl: "⌞", - construction: "🚧", - cong: "≅", - con: "⬝", - complement: "∁", - comp: "∘", - com: "ℂ", - coloneq: "≔", - colon: "₡", - copyright: "©", - coprod: "∐", - cdots: "⋯", - cdot: "⬝", - cd: "ḑ", - cib: "●", - ciw: "○", - "ci..": "◌", - "ci.": "◎", - ciO: "◯", - circeq: "≗", - circlearrowleft: "↺", - circlearrowright: "↻", - circledR: "®", - circledS: "Ⓢ", - circledast: "⊛", - circledcirc: "⊚", - circleddash: "⊝", - circ: "∘", - ci: "●", - centerdot: "·", - cent: "¢", - cedi: "₵", - celsius: "℃", - ce: "ȩ", - "c{h}": "ḩ", - "c{H}": "Ḩ", - "c{d}": "ḑ", - "c{D}": "Ḑ", - "c{e}": "ȩ", - "c{E}": "Ȩ", - "c{t}": "ţ", - "c{T}": "Ţ", - "c{s}": "ş", - "c{S}": "Ş", - "c{r}": "ŗ", - "c{R}": "Ŗ", - "c{n}": "ņ", - "c{N}": "Ņ", - "c{l}": "ļ", - "c{L}": "Ļ", - "c{k}": "ķ", - "c{K}": "Ķ", - "c{g}": "ģ", - "c{G}": "Ģ", - "c{c}": "ç", - "c{C}": "Ç", - "c{}": "¸", - checkmark: "✓", - chi: "χ", - ch: "ḩ", - cH: "Ḩ", - "c\\u{e}": "ḝ", - "c\\u{E}": "Ḝ", - "c\\ue": "ḝ", - "c\\uE": "Ḝ", - "c\\'{c}": "ḉ", - "c\\'{C}": "Ḉ", - "c\\'c": "ḉ", - "c\\'C": "Ḉ", - cD: "Ḑ", - cE: "Ȩ", - ct: "ţ", - cT: "Ţ", - cs: "ş", - cS: "Ş", - cruzeiro: "₢", - cr: "ŗ", - cR: "Ŗ", - cn: "ņ", - cN: "Ņ", - cL: "Ļ", - ck: "ķ", - cK: "Ķ", - cg: "ģ", - cG: "Ģ", - cc: "ç", - cC: "Ç", - caution: "☡", - cap: "∩", - c: "⌜", - qed: "∎", - quad: " ", - xi: "ξ", - x: "×", - "+ ": "⊹", - "&": "⅋", - "b+": "⊞", - "b-": "⊟", - bx: "⊠", - "b.": "⊡", - bn: "ℕ", - bz: "ℤ", - bq: "ℚ", - brokenbar: "¦", - br: "ℝ", - bc: "ℂ", - bp: "ℙ", - bb: "𝔹", - bsum: "⅀", - b0: "𝟘", - b1: "𝟙", - b2: "𝟚", - b3: "𝟛", - b4: "𝟜", - b5: "𝟝", - b6: "𝟞", - b7: "𝟟", - b8: "𝟠", - b9: "𝟡", - bub: "•", - buw: "◦", - but: "‣", - bumpeq: "≏", - bu: "•", - biohazard: "☣", - bigcap: "⋂", - bigcirc: "◯", - bigcup: "⋃", - bigstar: "★", - bigtriangledown: "▽", - bigtriangleup: "△", - bigvee: "⋁", - bigwedge: "⋀", - beta: "β", - beth: "ℶ", - between: "≬", - because: "∵", - backcong: "≌", - backepsilon: "∍", - backprime: "‵", - backsimeq: "⋍", - backsim: "∽", - barwedge: "⊼", - blacklozenge: "✦", - blacksquare: "▪", - blacksmiley: "☻", - blacktriangledown: "▾", - blacktriangleleft: "◂", - blacktriangleright: "▸", - blacktriangle: "▴", - bot: "⊥", - bowtie: "⋈", - boxminus: "⊟", - boxplus: "⊞", - boxtimes: "⊠", - b: "β", - join: "⋈", - "r-2": "⇉", - "r-3": "⇶", - "r-l-": "⇄", - "r--": "⟶", - "r-n": "↛", - "r-|": "↦", - "r->": "↣", - "r-o": "⊸", - "r-": "→", - "r==": "⇛", - "r=n": "⇏", - "r=": "⇒", - "r~": "↝", - "rr-": "↠", - rrbracket: "〛", - reb: "▬", - rew: "▭", - real: "ℝ", - registered: "®", - re: "▬", - rbag: "⟆", - rat: "ℚ", - radioactive: "☢", - rangle: "⟩", - rq: "’", - rial: "﷼", - rightarrowtail: "↣", - rightarrow: "→", - rightharpoondown: "⇁", - rightharpoonup: "⇀", - rightleftarrows: "⇄", - rightleftharpoons: "⇌", - rightrightarrows: "⇉", - rightthreetimes: "⋌", - risingdotseq: "≓", - ruble: "₽", - rupee: "₨", - rho: "ρ", - rhd: "▷", - rceil: "⌉", - rfloor: "⌋", - rtimes: "⋊", - rdq: "”", - rdata: "》", - r: "→", - functor: "⇒", - fun: "λ", - "f<<": "«", - "f<": "‹", - "f>>": "»", - "f>": "›", - frac12: "½", - frac13: "⅓", - frac14: "¼", - frac15: "⅕", - frac16: "⅙", - frac18: "⅛", - frac1: "⅟", - frac23: "⅔", - frac25: "⅖", - frac34: "¾", - frac35: "⅗", - frac38: "⅜", - frac45: "⅘", - frac56: "⅚", - frac58: "⅝", - frac78: "⅞", - frac: "¼", - frown: "⌢", - frqq: "»", - frq: "›", - female: "♀", - fei: "ϥ", - facsimile: "℻", - fallingdotseq: "≒", - flat: "♭", - flqq: "«", - flq: "‹", - forall: "∀", - ")b": "⟆", - ")": ")", - "[[": "⟦", - "]]": "⟧", - "{{": "⦃", - "}}": "⦄", - Xi: "Ξ", - X: "⨯", - "'{w}": "ẃ", - "'{W}": "Ẃ", - "'{p}": "ṕ", - "'{P}": "Ṕ", - "'{m}": "ḿ", - "'{M}": "Ḿ", - "'{k}": "ḱ", - "'{K}": "Ḱ", - "'{g}": "ǵ", - "'{G}": "Ǵ", - "'{z}": "ź", - "'{Z}": "Ź", - "'{s}": "ś", - "'{S}": "Ś", - "'{r}": "ŕ", - "'{R}": "Ŕ", - "'{n}": "ń", - "'{N}": "Ń", - "'{l}": "ĺ", - "'{L}": "Ĺ", - "'{c}": "ć", - "'{C}": "Ć", - "'{y}": "ý", - "'{u}": "ú", - "'{o}": "ó", - "'{i}": "í", - "'{e}": "é", - "'{a}": "á", - "'{Y}": "Ý", - "'{U}": "Ú", - "'{O}": "Ó", - "'{I}": "Í", - "'{E}": "É", - "'{A}": "Á", - "'{}": "´", - "'{\\AE}": "Ǽ", - "'{\\ae}": "ǽ", - "'{\\O}": "Ǿ", - "'{\\o}": "ǿ", - "'w": "ẃ", - "'W": "Ẃ", - "'\\.{s}": "ṥ", - "'\\.{S}": "Ṥ", - "'\\.s": "ṥ", - "'\\.S": "Ṥ", - "'\\AE": "Ǽ", - "'\\ae": "ǽ", - "'\\O": "Ǿ", - "'\\o": "ǿ", - "'p": "ṕ", - "'P": "Ṕ", - "'m": "ḿ", - "'M": "Ḿ", - "'k": "ḱ", - "'K": "Ḱ", - "'g": "ǵ", - "'G": "Ǵ", - "'z": "ź", - "'Z": "Ź", - "'s": "ś", - "'S": "Ś", - "'r": "ŕ", - "'R": "Ŕ", - "'n": "ń", - "'N": "Ń", - "'l": "ĺ", - "'L": "Ĺ", - "'c": "ć", - "'C": "Ć", - "'y": "ý", - "'u": "ú", - "'o": "ó", - "'i": "í", - "'e": "é", - "'a": "á", - "'Y": "Ý", - "'U": "Ú", - "'O": "Ó", - "'I": "Í", - "'E": "É", - "'A": "Á", - "'": "′", - "`{y}": "ỳ", - "`{Y}": "Ỳ", - "`{w}": "ẁ", - "`{W}": "Ẁ", - "`{n}": "ǹ", - "`{N}": "Ǹ", - "`{u}": "ù", - "`{o}": "ò", - "`{i}": "ì", - "`{e}": "è", - "`{a}": "à", - "`{U}": "Ù", - "`{O}": "Ò", - "`{I}": "Ì", - "`{E}": "È", - "`{A}": "À", - "`y": "ỳ", - "`Y": "Ỳ", - "`w": "ẁ", - "`W": "Ẁ", - "`n": "ǹ", - "`N": "Ǹ", - "`u": "ù", - "`o": "ò", - "`i": "ì", - "`e": "è", - "`a": "à", - "`U": "Ù", - "`O": "Ò", - "`I": "Ì", - "`E": "È", - "`A": "À", - "`": "‵", - Nat: "ℕ", - "N-2": "ℕ₋₂", - "N-1": "ℕ₋₁", - Nu: "Ν", - N: "ℕ", - Zeta: "Ζ", - Z: "ℤ", - Rat: "ℚ", - Real: "ℝ", - Re: "ℜ", - Rho: "Ρ", - Rightarrow: "⇒", - Rrightarrow: "⇛", - Rsh: "↱", - R: "ℝ", - Q: "ℚ", - Fei: "Ϥ", - Frowny: "☹", - F: "𝔽", - "H{u}": "ű", - "H{U}": "Ű", - "H{O}": "Ő", - "H{}": "˝", - "H{o}": "ő", - Hu: "ű", - HU: "Ű", - Hori: "Ϩ", - Ho: "ő", - HO: "Ő", - Heta: "Ͱ", - H: "ℍ", - Khei: "Ϧ", - Koppa: "Ϟ", - Kappa: "Κ", - K: "𝕂", - "#": "♯", - "^i": "ⁱ", - "^o_": "º", - "^o": "ᵒ", - "^--": "̅", - "^-": "⁻", - "^~": "̃", - "^.": "̇", - "^l-": "⃖", - "^lr": "⃡", - "^l": "⃖", - "^r-": "⃗", - "^r": "⃗", - "^^": "̂", - "^v": "̌", - "^a_": "ª", - "^a": "â", - "^\\d{o}": "ộ", - "^\\d{O}": "Ộ", - "^\\d{e}": "ệ", - "^\\d{E}": "Ệ", - "^\\d{a}": "ậ", - "^\\d{A}": "Ậ", - "^\\dotless j with stroke": "ᶡ", - "^\\delta": "ᵟ", - "^\\~{o}": "ỗ", - "^\\~{O}": "Ỗ", - "^\\~{e}": "ễ", - "^\\~{E}": "Ễ", - "^\\~{a}": "ẫ", - "^\\~{A}": "Ẫ", - "^\\~o": "ỗ", - "^\\~O": "Ỗ", - "^\\~e": "ễ", - "^\\~E": "Ễ", - "^\\~a": "ẫ", - "^\\~A": "Ẫ", - "^\\`{o}": "ồ", - "^\\`{O}": "Ồ", - "^\\`{e}": "ề", - "^\\`{E}": "Ề", - "^\\`{a}": "ầ", - "^\\`{A}": "Ầ", - "^\\`o": "ồ", - "^\\`O": "Ồ", - "^\\`e": "ề", - "^\\`E": "Ề", - "^\\`a": "ầ", - "^\\`A": "Ầ", - "^\\'{o}": "ố", - "^\\'{O}": "Ố", - "^\\'{e}": "ế", - "^\\'{E}": "Ế", - "^\\'{a}": "ấ", - "^\\'{A}": "Ấ", - "^\\'o": "ố", - "^\\'O": "Ố", - "^\\'e": "ế", - "^\\'E": "Ế", - "^\\'a": "ấ", - "^\\'A": "Ấ", - "^\\u with left hook": "ꭟ", - "^\\u bar": "ᶶ", - "^\\upsilon": "ᶷ", - "^\\l with middle tilde": "ꭞ", - "^\\l with inverted lazy s": "ꭝ", - "^\\l with palatal hook": "ᶪ", - "^\\l with retroflex hook": "ᶩ", - "^\\ligature oe": "ꟹ", - "^\\heng": "ꭜ", - "^\\h hook": "ʱ", - "^\\h with hook": "ʱ", - "^\\H With Stroke": "ꟸ", - "^\\theta": "ᶿ", - "^\\turned v": "ᶺ", - "^\\turned m with long leg": "ᶭ", - "^\\turned m": "ᵚ", - "^\\turned h": "ᶣ", - "^\\turned alpha": "ᶛ", - "^\\turned ae": "ᵆ", - "^\\turned a": "ᵄ", - "^\\turned i": "ᵎ", - "^\\turned open e": "ᵌ", - "^\\turned r hook": "ʵ", - "^\\turned r with hook": "ʵ", - "^\\turned r": "ʴ", - "^\\t with palatal hook": "ᶵ", - "^\\top half o": "ᵔ", - "^\\ezh": "ᶾ", - "^\\esh": "ᶴ", - "^\\eth": "ᶞ", - "^\\eng": "ᵑ", - "^\\z with curl": "ᶽ", - "^\\z with retroflex hook": "ᶼ", - "^\\v with hook": "ᶹ", - "^\\capital u": "ᶸ", - "^\\capital n": "ᶰ", - "^\\capital l": "ᶫ", - "^\\capital i with stroke": "ᶧ", - "^\\capital inverted r": "ʶ", - "^\\capital i": "ᶦ", - "^\\c with curl": "ᶝ", - "^\\chi": "ᵡ", - "^\\s with hook": "ᶳ", - "^\\script g": "ᶢ", - "^\\schwa": "ᵊ", - "^\\sideways u": "ᵙ", - "^\\phi": "ᶲ", - "^\\barred o": "ᶱ", - "^\\beta": "ᵝ", - "^\\bottom half o": "ᵕ", - "^\\n with retroflex hook": "ᶯ", - "^\\n with left hook": "ᶮ", - "^\\m with hook": "ᶬ", - "^\\j with crossed-tail": "ᶨ", - "^\\iota": "ᶥ", - "^\\i with stroke": "ᶤ", - "^\\reversed open e": "ᶟ", - "^\\reversed glottal stop": "ˤ", - "^\\greek phi": "ᵠ", - "^\\greek gamma": "ᵞ", - "^\\gamma": "ˠ", - "^\\ain": "ᵜ", - "^\\alpha": "ᵅ", - "^\\open o": "ᵓ", - "^\\open e": "ᵋ", - "^\\Ou": "ᴽ", - "^\\Reversed N": "ᴻ", - "^\\Reversed E": "ᴲ", - "^\\Barred B": "ᴯ", - "^\\Ae": "ᴭ", - "^{z}": "ẑ", - "^{Z}": "Ẑ", - "^{y}": "ŷ", - "^{Y}": "Ŷ", - "^{w}": "ŵ", - "^{W}": "Ŵ", - "^{s}": "ŝ", - "^{S}": "Ŝ", - "^{SM}": "℠", - "^{j}": "ĵ", - "^{J}": "Ĵ", - "^{h}": "ĥ", - "^{H}": "Ĥ", - "^{g}": "ĝ", - "^{G}": "Ĝ", - "^{c}": "ĉ", - "^{C}": "Ĉ", - "^{u}": "û", - "^{o}": "ô", - "^{i}": "î", - "^{e}": "ê", - "^{a}": "â", - "^{U}": "Û", - "^{O}": "Ô", - "^{I}": "Î", - "^{E}": "Ê", - "^{A}": "Â", - "^{\\j}": "ĵ", - "^{TEL}": "℡", - "^{TM}": "™", - "^z": "ẑ", - "^Z": "Ẑ", - "^y": "ŷ", - "^Y": "Ŷ", - "^w": "ŵ", - "^W": "Ŵ", - "^s": "ŝ", - "^S": "Ŝ", - "^j": "ĵ", - "^J": "Ĵ", - "^h": "ĥ", - "^H": "Ĥ", - "^g": "ĝ", - "^G": "Ĝ", - "^c": "ĉ", - "^C": "Ĉ", - "^u": "û", - "^e": "ê", - "^U": "Û", - "^O": "Ô", - "^I": "Î", - "^E": "Ê", - "^A": "Â", - "^n": "ⁿ", - "^)": "⁾", - "^(": "⁽", - "^=": "⁼", - "^+": "⁺", - "^9": "⁹", - "^8": "⁸", - "^7": "⁷", - "^6": "⁶", - "^5": "⁵", - "^4": "⁴", - "^0": "⁰", - "^1": "¹", - "^3": "³", - "^2": "²", - "^V": "ⱽ", - "^f": "ᶠ", - "^t": "ᵗ", - "^p": "ᵖ", - "^m": "ᵐ", - "^k": "ᵏ", - "^d": "ᵈ", - "^b": "ᵇ", - "^T": "ᵀ", - "^R": "ᴿ", - "^P": "ᴾ", - "^N": "ᴺ", - "^M": "ᴹ", - "^L": "ᴸ", - "^K": "ᴷ", - "^D": "ᴰ", - "^B": "ᴮ", - "^x": "ˣ", - "^": "̂", - "!!": "‼", - "!?": "⁉", - "!": "¡", - "_--": "̲", - "_-": "₋", - "_~": "̰", - "_.": "̣", - _lr: "͍", - _l: "ₗ", - "_^": "̭", - _v: "̬", - _j: "ⱼ", - _t: "ₜ", - _s: "ₛ", - _p: "ₚ", - _n: "ₙ", - _m: "ₘ", - _k: "ₖ", - _h: "ₕ", - _x: "ₓ", - _o: "ₒ", - _e: "ₑ", - _a: "ₐ", - "_)": "₎", - "_(": "₍", - "_=": "₌", - "_+": "₊", - _9: "₉", - _8: "₈", - _7: "₇", - _6: "₆", - _5: "₅", - _4: "₄", - _3: "₃", - _2: "₂", - _1: "₁", - _0: "₀", - _u: "ᵤ", - _r: "ᵣ", - _i: "ᵢ", - San: "Ϻ", - Sampi: "Ϡ", - Sho: "Ϸ", - Shima: "Ϭ", - Shei: "Ϣ", - Stigma: "Ϛ", - Sigma: "Σ", - Subset: "⋐", - Supset: "⋑", - Smiley: "☺", - S: "Σ", - Psi: "Ψ", - Phi: "Φ", - Pi: "Π", - P: "Π", - MiA: "𝐴", - MiB: "𝐵", - MiC: "𝐶", - MiD: "𝐷", - MiE: "𝐸", - MiF: "𝐹", - MiG: "𝐺", - MiH: "𝐻", - MiI: "𝐼", - MiJ: "𝐽", - MiK: "𝐾", - MiL: "𝐿", - MiM: "𝑀", - MiN: "𝑁", - MiO: "𝑂", - MiP: "𝑃", - MiQ: "𝑄", - MiR: "𝑅", - MiS: "𝑆", - MiT: "𝑇", - MiU: "𝑈", - MiV: "𝑉", - MiW: "𝑊", - MiX: "𝑋", - MiY: "𝑌", - MiZ: "𝑍", - Mia: "𝑎", - Mib: "𝑏", - Mic: "𝑐", - Mid: "𝑑", - Mie: "𝑒", - Mif: "𝑓", - Mig: "𝑔", - Mii: "𝑖", - Mij: "𝑗", - Mik: "𝑘", - Mil: "𝑙", - Mim: "𝑚", - Min: "𝑛", - Mio: "𝑜", - Mip: "𝑝", - Miq: "𝑞", - Mir: "𝑟", - Mis: "𝑠", - Mit: "𝑡", - Miu: "𝑢", - Miv: "𝑣", - Miw: "𝑤", - Mix: "𝑥", - Miy: "𝑦", - Miz: "𝑧", - MIA: "𝑨", - MIB: "𝑩", - MIC: "𝑪", - MID: "𝑫", - MIE: "𝑬", - MIF: "𝑭", - MIG: "𝑮", - MIH: "𝑯", - MII: "𝑰", - MIJ: "𝑱", - MIK: "𝑲", - MIL: "𝑳", - MIM: "𝑴", - MIN: "𝑵", - MIO: "𝑶", - MIP: "𝑷", - MIQ: "𝑸", - MIR: "𝑹", - MIS: "𝑺", - MIT: "𝑻", - MIU: "𝑼", - MIV: "𝑽", - MIW: "𝑾", - MIX: "𝑿", - MIY: "𝒀", - MIZ: "𝒁", - MIa: "𝒂", - MIb: "𝒃", - MIc: "𝒄", - MId: "𝒅", - MIe: "𝒆", - MIf: "𝒇", - MIg: "𝒈", - MIh: "𝒉", - MIi: "𝒊", - MIj: "𝒋", - MIk: "𝒌", - MIl: "𝒍", - MIm: "𝒎", - MIn: "𝒏", - MIo: "𝒐", - MIp: "𝒑", - MIq: "𝒒", - MIr: "𝒓", - MIs: "𝒔", - MIt: "𝒕", - MIu: "𝒖", - MIv: "𝒗", - MIw: "𝒘", - MIx: "𝒙", - MIy: "𝒚", - MIz: "𝒛", - McA: "𝒜", - McB: "ℬ", - McC: "𝒞", - McD: "𝒟", - McE: "ℰ", - McF: "ℱ", - McG: "𝒢", - McH: "ℋ", - McI: "ℐ", - McJ: "𝒥", - McK: "𝒦", - McL: "ℒ", - McM: "ℳ", - McN: "𝒩", - McO: "𝒪", - McP: "𝒫", - McQ: "𝒬", - McR: "ℛ", - McS: "𝒮", - McT: "𝒯", - McU: "𝒰", - McV: "𝒱", - McW: "𝒲", - McX: "𝒳", - McY: "𝒴", - McZ: "𝒵", - Mca: "𝒶", - Mcb: "𝒷", - Mcc: "𝒸", - Mcd: "𝒹", - Mce: "ℯ", - Mcf: "𝒻", - Mcg: "ℊ", - Mch: "𝒽", - Mci: "𝒾", - Mcj: "𝒿", - Mck: "𝓀", - Mcl: "𝓁", - Mcm: "𝓂", - Mcn: "𝓃", - Mco: "ℴ", - Mcp: "𝓅", - Mcq: "𝓆", - Mcr: "𝓇", - Mcs: "𝓈", - Mct: "𝓉", - Mcu: "𝓊", - Mcv: "𝓋", - Mcw: "𝓌", - Mcx: "𝓍", - Mcy: "𝓎", - Mcz: "𝓏", - MCA: "𝓐", - MCB: "𝓑", - MCC: "𝓒", - MCD: "𝓓", - MCE: "𝓔", - MCF: "𝓕", - MCG: "𝓖", - MCH: "𝓗", - MCI: "𝓘", - MCJ: "𝓙", - MCK: "𝓚", - MCL: "𝓛", - MCM: "𝓜", - MCN: "𝓝", - MCO: "𝓞", - MCP: "𝓟", - MCQ: "𝓠", - MCR: "𝓡", - MCS: "𝓢", - MCT: "𝓣", - MCU: "𝓤", - MCV: "𝓥", - MCW: "𝓦", - MCX: "𝓧", - MCY: "𝓨", - MCZ: "𝓩", - MCa: "𝓪", - MCb: "𝓫", - MCc: "𝓬", - MCd: "𝓭", - MCe: "𝓮", - MCf: "𝓯", - MCg: "𝓰", - MCh: "𝓱", - MCi: "𝓲", - MCj: "𝓳", - MCk: "𝓴", - MCl: "𝓵", - MCm: "𝓶", - MCn: "𝓷", - MCo: "𝓸", - MCp: "𝓹", - MCq: "𝓺", - MCr: "𝓻", - MCs: "𝓼", - MCt: "𝓽", - MCu: "𝓾", - MCv: "𝓿", - MCw: "𝔀", - MCx: "𝔁", - MCy: "𝔂", - MCz: "𝔃", - MfA: "𝔄", - MfB: "𝔅", - MfC: "ℭ", - MfD: "𝔇", - MfE: "𝔈", - MfF: "𝔉", - MfG: "𝔊", - MfH: "ℌ", - MfI: "ℑ", - MfJ: "𝔍", - MfK: "𝔎", - MfL: "𝔏", - MfM: "𝔐", - MfN: "𝔑", - MfO: "𝔒", - MfP: "𝔓", - MfQ: "𝔔", - MfR: "ℜ", - MfS: "𝔖", - MfT: "𝔗", - MfU: "𝔘", - MfV: "𝔙", - MfW: "𝔚", - MfX: "𝔛", - MfY: "𝔜", - MfZ: "ℨ", - Mfa: "𝔞", - Mfb: "𝔟", - Mfc: "𝔠", - Mfd: "𝔡", - Mfe: "𝔢", - Mff: "𝔣", - Mfg: "𝔤", - Mfh: "𝔥", - Mfi: "𝔦", - Mfj: "𝔧", - Mfk: "𝔨", - Mfl: "𝔩", - Mfm: "𝔪", - Mfn: "𝔫", - Mfo: "𝔬", - Mfp: "𝔭", - Mfq: "𝔮", - Mfr: "𝔯", - Mfs: "𝔰", - Mft: "𝔱", - Mfu: "𝔲", - Mfv: "𝔳", - Mfw: "𝔴", - Mfx: "𝔵", - Mfy: "𝔶", - Mfz: "𝔷", - Mu: "Μ", - " ": " ", - yen: "¥", - y: "ɏ", - '"{t}': "ẗ", - '"{x}': "ẍ", - '"{X}': "Ẍ", - '"{w}': "ẅ", - '"{W}': "Ẅ", - '"{h}': "ḧ", - '"{H}': "Ḧ", - '"{Y}': "Ÿ", - '"{y}': "ÿ", - '"{u}': "ü", - '"{o}': "ö", - '"{i}': "ï", - '"{e}': "ë", - '"{a}': "ä", - '"{U}': "Ü", - '"{O}': "Ö", - '"{I}': "Ï", - '"{E}': "Ë", - '"{A}': "Ä", - '"{}': "¨", - '"t': "ẗ", - '"x': "ẍ", - '"X': "Ẍ", - '"w': "ẅ", - '"W': "Ẅ", - "\"\\'{i}": "ḯ", - "\"\\'{I}": "Ḯ", - "\"\\'{u}": "ǘ", - "\"\\'{U}": "Ǘ", - "\"\\'i": "ḯ", - "\"\\'I": "Ḯ", - "\"\\'u": "ǘ", - "\"\\'U": "Ǘ", - '"\\={o}': "ȫ", - '"\\={O}': "Ȫ", - '"\\={a}': "ǟ", - '"\\={A}': "Ǟ", - '"\\={u}': "ǖ", - '"\\={U}': "Ǖ", - '"\\=o': "ȫ", - '"\\=O': "Ȫ", - '"\\=a': "ǟ", - '"\\=A': "Ǟ", - '"\\=u': "ǖ", - '"\\=U': "Ǖ", - '"\\`{u}': "ǜ", - '"\\`{U}': "Ǜ", - '"\\`u': "ǜ", - '"\\`U': "Ǜ", - '"\\v{u}': "ǚ", - '"\\v{U}': "Ǚ", - '"\\vu': "ǚ", - '"\\vU': "Ǚ", - '"h': "ḧ", - '"H': "Ḧ", - '"Y': "Ÿ", - '"y': "ÿ", - '"u': "ü", - '"o': "ö", - '"i': "ï", - '"e': "ë", - '"a': "ä", - '"U': "Ü", - '"O': "Ö", - '"I': "Ï", - '"E': "Ë", - '"A': "Ä", - '"`': "„", - "\"'": "“", - '"<': "«", - '">': "»", - '"': "̈", - "v\\.{s}": "ṧ", - "v\\.{S}": "Ṧ", - "v\\.s": "ṧ", - "v\\.S": "Ṧ", - "v{h}": "ȟ", - "v{H}": "Ȟ", - "v{j}": "ǰ", - "v{k}": "ǩ", - "v{K}": "Ǩ", - "v{g}": "ǧ", - "v{G}": "Ǧ", - "v{u}": "ǔ", - "v{U}": "Ǔ", - "v{o}": "ǒ", - "v{O}": "Ǒ", - "v{i}": "ǐ", - "v{I}": "Ǐ", - "v{a}": "ǎ", - "v{A}": "Ǎ", - "v{z}": "ž", - "v{Z}": "Ž", - "v{t}": "ť", - "v{T}": "Ť", - "v{s}": "š", - "v{S}": "Š", - "v{r}": "ř", - "v{R}": "Ř", - "v{n}": "ň", - "v{N}": "Ň", - "v{l}": "ľ", - "v{L}": "Ľ", - "v{e}": "ě", - "v{E}": "Ě", - "v{d}": "ď", - "v{D}": "Ď", - "v{c}": "č", - "v{C}": "Č", - "v{}": "ˇ", - "v{\\i}": "ǐ", - "v{\\j}": "ǰ", - vh: "ȟ", - vH: "Ȟ", - vj: "ǰ", - vk: "ǩ", - vK: "Ǩ", - vg: "ǧ", - vG: "Ǧ", - vu: "ǔ", - vU: "Ǔ", - vo: "ǒ", - vO: "Ǒ", - vi: "ǐ", - vI: "Ǐ", - varrho: "ϱ", - varkappa: "ϰ", - varkai: "ϗ", - varpi: "ϖ", - varphi: "ϕ", - varprime: "′", - varpropto: "∝", - vartheta: "ϑ", - vartriangleleft: "⊲", - vartriangleright: "⊳", - varbeta: "ϐ", - varsigma: "ς", - va: "ǎ", - vA: "Ǎ", - vz: "ž", - vZ: "Ž", - vt: "ť", - vT: "Ť", - vs: "š", - vS: "Š", - vr: "ř", - vR: "Ř", - vn: "ň", - vN: "Ň", - vl: "ľ", - vL: "Ľ", - veebar: "⊻", - vee: "∨", - ve: "ě", - vE: "Ě", - vdash: "⊢", - vdots: "⋮", - vd: "ď", - vDash: "⊨", - vD: "Ď", - vc: "č", - vC: "Č", - v: "̌", - "k\\={o}": "ǭ", - "k\\={O}": "Ǭ", - "k\\=o": "ǭ", - "k\\=O": "Ǭ", - "k{o}": "ǫ", - "k{O}": "Ǫ", - "k{u}": "ų", - "k{U}": "Ų", - "k{i}": "į", - "k{I}": "Į", - "k{e}": "ę", - "k{E}": "Ę", - "k{a}": "ą", - "k{A}": "Ą", - "k{}": "˛", - koppa: "ϟ", - ko: "ǫ", - kO: "Ǫ", - ku: "ų", - kU: "Ų", - kip: "₭", - ki: "į", - kI: "Į", - kelvin: "K", - ke: "ę", - kE: "Ę", - kappa: "κ", - ka: "ą", - kA: "Ą", - khei: "ϧ", - k: "̨", - ",": " ", - "/": "‌", - ";": " ", - warning: "⚠", - won: "₩", - wedge: "∧", - wp: "℘", - wr: "≀", - Dei: "Ϯ", - Delta: "Δ", - Digamma: "Ϝ", - Diamond: "◇", - Downarrow: "⇓", - DH: "Ð", - zeta: "ζ", - Eta: "Η", - Epsilon: "Ε", - Beta: "Β", - Box: "□", - Bumpeq: "≎", - bbA: "𝔸", - bbB: "𝔹", - bbC: "ℂ", - bbD: "𝔻", - bbE: "𝔼", - bbF: "𝔽", - bbG: "𝔾", - bbH: "ℍ", - bbI: "𝕀", - bbJ: "𝕁", - bbK: "𝕂", - bbL: "𝕃", - bbM: "𝕄", - bbN: "ℕ", - bbO: "𝕆", - bbP: "ℙ", - bbQ: "ℚ", - bbR: "ℝ", - bbS: "𝕊", - bbT: "𝕋", - bbU: "𝕌", - bbV: "𝕍", - bbW: "𝕎", - bbX: "𝕏", - bbY: "𝕐", - bbZ: "ℤ", - Yot: "Ϳ", - Join: "⋈", - Vdash: "⊩", - Vert: "‖", - Vvdash: "⊪", -}; diff --git a/src/packages/frontend/frame-editors/lean-editor/tab-completion.ts b/src/packages/frontend/frame-editors/lean-editor/tab-completion.ts deleted file mode 100644 index 23dba1a8b3..0000000000 --- a/src/packages/frontend/frame-editors/lean-editor/tab-completion.ts +++ /dev/null @@ -1,77 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -/* -Register a CodeMirror hinter for the mode with name 'lean'. - -*/ - -import * as CodeMirror from "codemirror"; - -import { Completion } from "./types"; - -import { Actions } from "./actions"; - -import { completions } from "@cocalc/frontend/codemirror/mode/lean"; - -interface CMCompletion { - text: string; - displayText: string; -} - -async function leanHint( - cm: CodeMirror.Editor -): Promise<{ list: CMCompletion[]; from: any; to: any } | void> { - const cur = cm.getDoc().getCursor(), - token = cm.getTokenAt(cur); - - const set: any = {}; - const list: CMCompletion[] = []; - function include(words: string[]): void { - for (const word of words) { - if (!set[word]) { - set[word] = true; - list.push({ text: word, displayText: `◇ ${word}` }); - } - } - } - - // First start with list of completions coming from - // the syntax highlighting mode. - let t = (CodeMirror as any).hint.anyword(cm); - if (t != null && t.list != null) { - include(t.list); - } - - // We have to also do this, since the above misses words that haven't already been highlighted! - t = (CodeMirror as any).hint.fromList(cm, { words: completions }); - if (t != null && t.list != null) { - include(t.list); - } - - list.sort(); - - // completions coming from backend LEAN server. - if ((cm as any).cocalc_actions !== undefined) { - const actions: Actions = (cm as any).cocalc_actions; - - const resp: Completion[] = await actions.complete(cur.line, cur.ch); - - // First show those that match token.string, then show the rest. - for (let i = 0; i < resp.length; i++) { - const { text, type } = resp[i]; - const displayText = `▣ ${text} : ${type}`; - list.push({ text, displayText }); - } - } - - return { - list, - from: CodeMirror.Pos(cur.line, token.start), - to: CodeMirror.Pos(cur.line, token.end), - }; -} - -CodeMirror.registerHelper("hint", "lean", leanHint); diff --git a/src/packages/frontend/frame-editors/lean-editor/types.ts b/src/packages/frontend/frame-editors/lean-editor/types.ts deleted file mode 100644 index 99b264aad1..0000000000 --- a/src/packages/frontend/frame-editors/lean-editor/types.ts +++ /dev/null @@ -1,27 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -export interface Task { - desc: string; - end_pos_col: number; - end_pos_line: number; - pos_col: number; - pos_line: number; -} - -export interface Message { - caption: string; - end_pos_col: number; - end_pos_line: number; - pos_col: number; - pos_line: number; - severity: string; - text: string; -} - -export interface Completion { - text: string; - type: string; -} diff --git a/src/packages/frontend/frame-editors/register.ts b/src/packages/frontend/frame-editors/register.ts index 31aceb4180..912bc92be7 100644 --- a/src/packages/frontend/frame-editors/register.ts +++ b/src/packages/frontend/frame-editors/register.ts @@ -19,7 +19,6 @@ import "./pdf-editor/register"; import "./terminal-editor/register"; import "./x11-editor/register"; -import "./lean-editor/register"; import "./jupyter-editor/register"; import "./time-travel-editor/register"; import "./course-editor/register"; diff --git a/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts b/src/packages/frontend/frame-editors/terminal-editor/conat-terminal.ts similarity index 60% rename from src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts rename to src/packages/frontend/frame-editors/terminal-editor/conat-terminal.ts index c07c3b059a..015c55fb2f 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/conat-terminal.ts @@ -2,21 +2,23 @@ import { webapp_client } from "@cocalc/frontend/webapp-client"; import { EventEmitter } from "events"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { delay } from "awaiting"; -import { type DStream } from "@cocalc/nats/sync/dstream"; +import { type DStream } from "@cocalc/conat/sync/dstream"; import { createTerminalClient, type TerminalServiceApi, createBrowserService, SIZE_TIMEOUT_MS, createBrowserClient, -} from "@cocalc/nats/service/terminal"; -import { NATS_OPEN_FILE_TOUCH_INTERVAL } from "@cocalc/util/nats"; +} from "@cocalc/conat/service/terminal"; +import { CONAT_OPEN_FILE_TOUCH_INTERVAL } from "@cocalc/util/conat"; +import { until } from "@cocalc/util/async-utils"; type State = "disconnected" | "init" | "running" | "closed"; -export class NatsTerminalConnection extends EventEmitter { +export class ConatTerminal extends EventEmitter { private project_id: string; private path: string; + private termPath: string; public state: State = "init"; private stream?: DStream; private terminalResize; @@ -27,36 +29,47 @@ export class NatsTerminalConnection extends EventEmitter { private options?; private writeQueue: string = ""; private ephemeral?: boolean; + private computeServers?; constructor({ project_id, path, + termPath, terminalResize, openPaths, closePaths, options, measureSize, + ephemeral, }: { project_id: string; path: string; + termPath: string; terminalResize; openPaths; closePaths; options?; measureSize?; + ephemeral?: boolean; }) { super(); + this.ephemeral = ephemeral; this.project_id = project_id; this.path = path; + this.termPath = termPath; this.options = options; - this.touchLoop({ project_id, path }); + this.touchLoop({ project_id, path: termPath }); this.sizeLoop(measureSize); - this.api = createTerminalClient({ project_id, path }); + this.api = createTerminalClient({ project_id, termPath }); this.createBrowserService(); this.terminalResize = terminalResize; this.openPaths = openPaths; this.closePaths = closePaths; - webapp_client.nats_client.on("connected", this.clearWriteQueue); + webapp_client.conat_client.on("connected", this.clearWriteQueue); + this.computeServers = webapp_client.project_client.computeServers( + this.project_id, + ); + this.computeServers?.on("change", this.handleComputeServersChange); } clearWriteQueue = () => { @@ -71,12 +84,13 @@ export class NatsTerminalConnection extends EventEmitter { }; write = async (data) => { - if (this.state == "init" || this.state == "closed") { - // ignore initial data while initializing. - // This is the trick to avoid "junk characters" on refresh/reconnect. + if (this.state == "closed") { return; } if (this.state == "disconnected") { + if (typeof data == "string") { + this.writeQueue += data; + } await this.init(); return; } @@ -143,7 +157,7 @@ export class NatsTerminalConnection extends EventEmitter { if (this.state == ("closed" as State)) { break; } - await delay(NATS_OPEN_FILE_TOUCH_INTERVAL); + await delay(CONAT_OPEN_FILE_TOUCH_INTERVAL); } }; @@ -155,7 +169,14 @@ export class NatsTerminalConnection extends EventEmitter { }; close = async () => { - webapp_client.nats_client.removeListener("connected", this.clearWriteQueue); + webapp_client.conat_client.removeListener( + "connected", + this.clearWriteQueue, + ); + this.computeServers?.removeListener( + "change", + this.handleComputeServersChange, + ); this.stream?.close(); delete this.stream; this.service?.close(); @@ -173,77 +194,110 @@ export class NatsTerminalConnection extends EventEmitter { this.close(); }; - private start = reuseInFlight(async () => { + // try to get project/compute_server to start the corresponding + // terminal session on the backend. Keeps retrying until either + // this object is closed or it succeeds. + public start = reuseInFlight(async () => { this.setState("init"); - let maxWait = 5000; - while (true) { - try { - if (this.state == "closed") { - return; - } - const { success, note, ephemeral } = await this.api.create({ - ...this.options, - ephemeral: true, + await until( + async () => { + if (this.state == "closed") return true; + const compute_server_id = + (await webapp_client.project_client.getServerIdForPath({ + project_id: this.project_id, + path: this.termPath, + })) ?? 0; + const api = webapp_client.conat_client.projectApi({ + project_id: this.project_id, + compute_server_id, }); - this.ephemeral = ephemeral; - if (!success) { - throw Error(`failed to create terminal -- ${note}`); - } - return; - } catch (err) { - console.log(`Warning -- ${err} (will retry)`); try { - await this.api.nats.waitFor({ maxWait }); + await api.editor.createTerminalService(this.termPath, { + ...this.options, + ephemeral: this.ephemeral, + path: this.path, + }); + return true; } catch (err) { - maxWait = Math.min(15000, 1.3 * maxWait); - console.log(`WARNING -- waiting for terminal server -- ${err}`); - await delay(3000); + console.log(`WARNING: starting terminal -- ${err} (will retry)`); + return false; } - } - } + }, + { start: 2000, decay: 1.3, max: 15000 }, + ); }); - private getStream = async () => { - const { nats_client } = webapp_client; - return await nats_client.dstream({ - name: `terminal-${this.path}`, - project_id: this.project_id, - ephemeral: this.ephemeral, - }); + private handleComputeServersChange = ({ path }) => { + if (path != this.termPath) { + return; + } + this.start(); }; - init = async () => { - this.setState("init"); - await this.start(); + private getStream = async () => { + if (this.stream != null) { + this.stream.close(); + delete this.stream; + } if (this.state == "closed") { return; } - if (this.stream != null) { + this.stream = await webapp_client.conat_client.dstream({ + name: `terminal-${this.termPath}`, + project_id: this.project_id, + ephemeral: this.ephemeral, + }); + if (this.state == ("closed" as any)) { this.stream.close(); delete this.stream; + return; } - this.stream = await this.getStream(); - this.consumeDataStream(); + await this.consumeDataStream(); }; - private handleStreamData = (data) => { - if (data) { - this.emit("data", data); + init = reuseInFlight(async () => { + await Promise.all([this.start(), this.getStream()]); + await this.setReady(); + }); + + private seq: number = 0; + private incoming: { [seq: number]: string } | null = null; + private handleStreamData = (data, seq) => { + if (!this.seq || this.seq + 1 == seq) { + // got the correct seq + this.seq = seq; + if (this.incoming == null) { + // easy case -- nothing out of order queued up + this.emit("data", data); + return; + } + // broadcast seq and anything next after it that we + // have in our incoming queue. + this.incoming[seq] = data; + let s = seq; + while (this.incoming[s] !== undefined) { + this.emit("data", this.incoming[s]); + this.seq = s; + delete this.incoming[s]; + s += 1; + } + return; + } else { + // got something out of order -- save it to incoming queue + if (this.incoming == null) { + this.incoming = {}; + } + this.incoming[seq] = data; } }; - private consumeDataStream = () => { + private consumeDataStream = async () => { if (this.stream == null) { return; } const initData = this.stream.getAll().join(""); - this.handleStreamData(initData); - this.setReady(); + this.emit("init", initData); this.stream.on("change", this.handleStreamData); - if (this.writeQueue) { - // causes anything in queue to be sent and queue to be cleared: - this.write(""); - } }; private setReady = async () => { @@ -252,12 +306,16 @@ export class NatsTerminalConnection extends EventEmitter { await delay(500); this.setState("running"); this.emit("ready"); + if (this.writeQueue) { + // causes anything in queue to be sent and queue to be cleared: + this.write(""); + } }; private browserClient = () => { return createBrowserClient({ project_id: this.project_id, - path: this.path, + termPath: this.termPath, }); }; @@ -289,7 +347,7 @@ export class NatsTerminalConnection extends EventEmitter { }; this.service = await createBrowserService({ project_id: this.project_id, - path: this.path, + termPath: this.termPath, impl, }); }; diff --git a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts index abbe4a4adb..701ea0cac1 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts @@ -14,11 +14,11 @@ extra support for being connected to: import { callback, delay } from "awaiting"; import { Map } from "immutable"; import { debounce } from "lodash"; -import { Terminal as XTerminal } from "xterm"; -import { FitAddon } from "xterm-addon-fit"; -import { WebLinksAddon } from "xterm-addon-web-links"; -import { WebglAddon } from "xterm-addon-webgl"; -import "xterm/css/xterm.css"; +import { Terminal as XTerminal } from "@xterm/xterm"; +import { FitAddon } from "@xterm/addon-fit"; +import { WebLinksAddon } from "@xterm/addon-web-links"; +import { WebglAddon } from "@xterm/addon-webgl"; +import "@xterm/xterm/css/xterm.css"; import { webapp_client } from "@cocalc/frontend/webapp-client"; import { ProjectActions, redux } from "@cocalc/frontend/app-framework"; import { get_buffer, set_buffer } from "@cocalc/frontend/copy-paste-buffer"; @@ -37,7 +37,7 @@ import { ConnectedTerminalInterface } from "./connected-terminal-interface"; import { open_init_file } from "./init-file"; import { setTheme } from "./themes"; import { modalParams } from "@cocalc/frontend/compute/select-server-for-file"; -import { NatsTerminalConnection } from "./nats-terminal-connection"; +import { ConatTerminal } from "./conat-terminal"; import { termPath } from "@cocalc/util/terminal/names"; declare const $: any; @@ -46,9 +46,8 @@ declare const $: any; const SCROLLBACK = 5000; const MAX_HISTORY_LENGTH = 100 * SCROLLBACK; -const MAX_DELAY = 10000; +const MAX_DELAY = 15000; -// See https://github.com/sagemathinc/cocalc/issues/8330 const ENABLE_WEBGL = false; interface Path { @@ -64,7 +63,7 @@ export class Terminal { private terminal_settings: Map; private project_id: string; private path: string; - private term_path: string; + private termPath: string; private id: string; readonly rendererType: "dom" | "canvas"; private terminal: XTerminal; @@ -91,7 +90,7 @@ export class Terminal { private last_geom: { rows: number; cols: number } | undefined; private resize_after_no_ignore: { rows: number; cols: number } | undefined; private last_active: number = 0; - private conn?: NatsTerminalConnection; + private conn?: ConatTerminal; private touch_interval; public is_visible: boolean = false; @@ -105,6 +104,9 @@ export class Terminal { private webLinksAddon: WebLinksAddon; private render_done: Function[] = []; + private ignoreData: boolean = false; + + private firstOpen = true; constructor( actions: Actions, @@ -133,7 +135,7 @@ export class Terminal { const cmd = command ? "-" + replace_all(command, "/", "-") : ""; // This is the one and only place number is used. // It's very important though. - this.term_path = termPath({ path: this.path, number, cmd }); + this.termPath = termPath({ path: this.path, number, cmd }); this.id = id; this.terminal = new XTerminal(this.get_xtermjs_options()); @@ -221,7 +223,9 @@ export class Terminal { }; close = (): void => { - this.assert_not_closed(); + if (this.state === "closed") { + return; + } this.set_connection_status("disconnected"); this.state = "closed"; clearInterval(this.touch_interval); @@ -282,12 +286,19 @@ export class Terminal { } this.ignore_terminal_data = true; this.set_connection_status("connecting"); - await this.configureComputeServerId(); + try { + await this.configureComputeServerId(); + } catch { + // expected if the project tab closes right when the terminal is + // being created. + return; + } if (this.state == "closed") { return; } - const conn = new NatsTerminalConnection({ - path: this.term_path, + const conn = new ConatTerminal({ + termPath: this.termPath, + path: this.path, project_id: this.project_id, terminalResize: this.terminal_resize, openPaths: this.open_paths, @@ -307,11 +318,19 @@ export class Terminal { this.actions.set_terminal_cwd(this.id, cwd); }); conn.on("data", this.handleDataFromProject); + conn.on("init", async (data) => { + // during init we write a bunch of data to the terminal (everything + // so far), and the terminal would respond to some of that data with + // control codes. We thus set ignoreData:true, so that during the + // parsing of this data by the browser terminal, those control codes + // are ignored. Not doing this properly was the longterm source of + // control code corruption in the terminal. + await this.render(data, { ignoreData: true }); + }); conn.once("ready", () => { delete this.last_geom; this.ignore_terminal_data = false; this.set_connection_status("connected"); - this.scroll_to_bottom(); this.terminal.refresh(0, this.terminal.rows - 1); this.init_keyhandler(); this.measureSize(); @@ -322,6 +341,10 @@ export class Terminal { this.conn_write_buffer.length = 0; } } + if (this.firstOpen) { + this.firstOpen = false; + this.project_actions.log_opened_time(this.path); + } }); if (endswith(this.path, ".term")) { touchPath(this.project_id, this.path); // no need to await @@ -351,6 +374,8 @@ export class Terminal { this.lastSend = Date.now(); }; + // this should never ever be necessary. It's a just-in-case things + // were myseriously totally broken measure... private reconnectIfNotResponding = async () => { while (this.state != "closed") { if (this.lastSend - this.lastReceive >= MAX_DELAY) { @@ -361,7 +386,6 @@ export class Terminal { }; private handleDataFromProject = (data: any): void => { - //console.log("data", data); this.assert_not_closed(); if (!data || typeof data != "string") { return; @@ -379,7 +403,13 @@ export class Terminal { this.project_actions.flag_file_activity(this.path); }; - render = async (data: string): Promise => { + private render = async ( + data: string, + { ignoreData = false }: { ignoreData?: boolean } = {}, + ): Promise => { + if (data == null) { + return; + } this.assert_not_closed(); this.history += data; if (this.history.length > MAX_HISTORY_LENGTH) { @@ -388,10 +418,20 @@ export class Terminal { ); } try { - await this.terminal.write(data); + this.ignoreData = ignoreData; + // NOTE: terminal.write takes a cb but not in the way callback expects. + // Also, terminal.write is NOT async, which was a bug in this code for a while. + await callback((cb) => { + this.terminal.write(data, () => { + cb(); + }); + }); } catch (err) { console.warn(`issue writing data to terminal: ${data}`); + } finally { + this.ignoreData = false; } + if (this.state == "done") return; // tell anyone who waited for output coming back about this while (this.render_done.length > 0) { this.render_done.pop()?.(); @@ -421,7 +461,11 @@ export class Terminal { }; touch = async () => { + if (this.state === "closed") return; if (Date.now() - this.last_active < 70000) { + if (this.project_actions.isTabClosed()) { + return; + } touch_project(this.project_id, await this.getComputeServerId()); } }; @@ -613,6 +657,9 @@ export class Terminal { return; } const project_actions: ProjectActions = this.actions._get_project_actions(); + if (project_actions.isTabClosed()) { + return; + } let i = 0; let foreground = false; const compute_server_id = await this.getComputeServerId(); @@ -707,7 +754,7 @@ export class Terminal { init_terminal_data(): void { this.terminal.onData((data) => { - if (this.ignore_terminal_data && this.conn?.state == "init") { + if (this.ignoreData) { return; } this.conn_write(data); @@ -726,12 +773,15 @@ export class Terminal { } refresh(): void { + if (this.state === "closed") { + return; + } this.terminal.refresh(0, this.terminal.rows - 1); } async edit_init_script(): Promise { try { - await open_init_file(this.actions._get_project_actions(), this.term_path); + await open_init_file(this.actions._get_project_actions(), this.termPath); } catch (err) { if (this.state === "closed") { return; @@ -743,7 +793,7 @@ export class Terminal { popout(): void { this.actions ._get_project_actions() - .open_file({ path: this.term_path, foreground: true }); + .open_file({ path: this.termPath, foreground: true }); } set_font_size(font_size: number): void { @@ -788,6 +838,9 @@ export class Terminal { }; scroll_to_bottom = (): void => { + if (this.terminal == null) { + return; + } // Upstream bug workaround -- we scroll to top first, then bottom // entirely to workaround a bug. This is NOT fixed by the Oct 2018 // term.js release, despite it touching relevant code. @@ -799,7 +852,7 @@ export class Terminal { const computeServerAssociations = webapp_client.project_client.computeServers(this.project_id); return ( - (await computeServerAssociations.getServerIdForPath(this.term_path)) ?? 0 + (await computeServerAssociations.getServerIdForPath(this.termPath)) ?? 0 ); }; @@ -812,7 +865,7 @@ export class Terminal { const computeServerAssociations = webapp_client.project_client.computeServers(this.project_id); const cur = await computeServerAssociations.getServerIdForPath( - this.term_path, + this.termPath, ); if (cur != null) { // nothing to do -- it's already explicitly set. @@ -827,13 +880,13 @@ export class Terminal { await redux .getActions("page") .popconfirm( - modalParams({ current: 0, target: id, path: this.term_path }), + modalParams({ current: 0, target: id, path: this.termPath }), ) ) { // yes, switch it computeServerAssociations.connectComputeServerToPath({ id, - path: this.term_path, + path: this.termPath, }); await computeServerAssociations.save(); } @@ -846,8 +899,9 @@ async function touchPath(project_id: string, path: string): Promise { // Also this is in a separate function so we can await it and catch exception. try { await touch(project_id, path); - } catch (err) { - console.warn(`error touching ${path} -- ${err}`); + } catch { + // expected to fail, e.g., it will on compute server while waiting to switch + //console.warn(`error touching ${path} -- ${err}`); } } diff --git a/src/packages/frontend/frame-editors/terminal-editor/terminal.tsx b/src/packages/frontend/frame-editors/terminal-editor/terminal.tsx index 62abe9e38a..eb15026d08 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/terminal.tsx +++ b/src/packages/frontend/frame-editors/terminal-editor/terminal.tsx @@ -135,7 +135,7 @@ export const TerminalFrame: React.FC = React.memo((props: Props) => { return false; }); - terminalRef.current.scroll_to_bottom(); + // terminalRef.current.scroll_to_bottom(); } const set_font_size = throttle(() => { diff --git a/src/packages/frontend/frame-editors/terminal-editor/themes.ts b/src/packages/frontend/frame-editors/terminal-editor/themes.ts index 7ddfa17908..6c4431dd77 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/themes.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/themes.ts @@ -7,7 +7,7 @@ Our predefined terminal color themes. */ -import { ITheme, Terminal } from "xterm"; +import { ITheme, Terminal } from "@xterm/xterm"; import { color_themes } from "./theme-data"; export function background_color(theme_name: string): string { diff --git a/src/packages/frontend/frame-editors/terminal-editor/tour.tsx b/src/packages/frontend/frame-editors/terminal-editor/tour.tsx index fdf7b0841f..cd1174dac8 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/tour.tsx +++ b/src/packages/frontend/frame-editors/terminal-editor/tour.tsx @@ -209,7 +209,7 @@ export default function getTour(refs) { target: "kick_other_users_out", title: ( <> - Kick Other Users Out + Force Resize ), description: ( diff --git a/src/packages/frontend/history.ts b/src/packages/frontend/history.ts index ed975b2315..5698681a0b 100644 --- a/src/packages/frontend/history.ts +++ b/src/packages/frontend/history.ts @@ -152,20 +152,6 @@ export function load_target( case "settings": redux.getActions("page").set_active_tab("account", false); - - if (segments[1] === "billing") { - const actions = redux.getActions("billing"); - actions?.update_customer(); - redux.getActions("account").set_active_tab("billing"); - if (actions == null) { - // ugly temporary hack. - setTimeout(() => { - redux.getActions("billing")?.update_customer(); - }, 5000); - } - return; - } - const actions = redux.getActions("account"); actions.set_active_tab(segments[1]); actions.setFragment(Fragment.decode(hash)); diff --git a/src/packages/frontend/i18n/common.ts b/src/packages/frontend/i18n/common.ts index 90b4f19413..d7c9de034d 100644 --- a/src/packages/frontend/i18n/common.ts +++ b/src/packages/frontend/i18n/common.ts @@ -926,11 +926,11 @@ export const menu = defineMessages({ }, kick_other_users_out_label: { id: "menu.generic.kick_other_users_out.label", - defaultMessage: "Kick Other Users Out", + defaultMessage: "Force Resize", }, kick_other_users_out_button: { id: "menu.generic.kick_other_users_out.button", - defaultMessage: "Kick", + defaultMessage: "Resize", }, kick_other_users_out_title: { id: "menu.generic.kick_other_users_out.title", diff --git a/src/packages/frontend/i18n/trans/extracted.json b/src/packages/frontend/i18n/trans/extracted.json index a4cdd74d68..b611b34e37 100644 --- a/src/packages/frontend/i18n/trans/extracted.json +++ b/src/packages/frontend/i18n/trans/extracted.json @@ -1381,10 +1381,10 @@ "description": "Button label or menu entry for the 'Jupyter Kernel'. Keep its name 'Kernel' in all languages." }, "menu.generic.kick_other_users_out.button": { - "defaultMessage": "Kick" + "defaultMessage": "Resize" }, "menu.generic.kick_other_users_out.label": { - "defaultMessage": "Kick Other Users Out" + "defaultMessage": "Force Resize" }, "menu.generic.kick_other_users_out.title": { "defaultMessage": "Kick all other users out from this document. It will close in all other browsers." diff --git a/src/packages/frontend/jest.config.js b/src/packages/frontend/jest.config.js index 140b9467f2..548ce54df6 100644 --- a/src/packages/frontend/jest.config.js +++ b/src/packages/frontend/jest.config.js @@ -1,6 +1,7 @@ /** @type {import('ts-jest').JestConfigWithTsJest} */ module.exports = { - preset: 'ts-jest', - testEnvironment: 'node', - testMatch: ['**/?(*.)+(spec|test).ts?(x)'], + preset: "ts-jest", + testEnvironment: "node", + testMatch: ["**/?(*.)+(spec|test).ts?(x)"], + setupFiles: ["./test/setup.js"], }; diff --git a/src/packages/frontend/jupyter/browser-actions.ts b/src/packages/frontend/jupyter/browser-actions.ts index 3c99b517ec..6eb2561292 100644 --- a/src/packages/frontend/jupyter/browser-actions.ts +++ b/src/packages/frontend/jupyter/browser-actions.ts @@ -36,8 +36,6 @@ import { } from "@cocalc/util/misc"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { JUPYTER_CLASSIC_MODERN } from "@cocalc/util/theme"; -import type { ImmutableUsageInfo } from "@cocalc/util/types/project-usage-info"; -import { get_usage_info, UsageInfoWS } from "../project/websocket/usage-info"; import { cm_options } from "./cm_options"; import { ConfirmDialogOptions } from "./confirm-dialog"; import { parseHeadings } from "./contents"; @@ -56,6 +54,8 @@ import { JUPYTER_MIMETYPES } from "@cocalc/jupyter/util/misc"; import { parse } from "path"; import { syncdbPath } from "@cocalc/util/jupyter/names"; import getKernelSpec from "@cocalc/frontend/jupyter/kernelspecs"; +import { get as getUsageInfo } from "@cocalc/conat/project/usage-info"; +import { delay } from "awaiting"; // local cache: map project_id (string) -> kernels (immutable) let jupyter_kernels = Map(); @@ -66,7 +66,6 @@ export class JupyterActions extends JupyterActions0 { private cursor_manager: CursorManager; private account_change_editor_settings: any; private update_keyboard_shortcuts: any; - private usage_info?: UsageInfoWS; private syncdbPath: string; protected init2(): void { @@ -77,7 +76,7 @@ export class JupyterActions extends JupyterActions0 { cell_toolbar: this.get_local_storage("cell_toolbar"), }); - this.usage_info_handler = this.usage_info_handler.bind(this); + this.initUsageInfo(); const do_set = () => { if (this.syncdb == null || this._state === "closed") return; @@ -152,17 +151,12 @@ export class JupyterActions extends JupyterActions0 { // cell notebook that has nothing to do with nbgrader). this.nbgrader_actions.update_metadata(); } - - const usage_info = (this.usage_info = get_usage_info(this.project_id)); - usage_info.watch(this.path); - const key = usage_info.event_key(this.path); - usage_info.on(key, this.usage_info_handler); }); // Put an entry in the project log once the jupyter notebook gets opened. // NOTE: Obviously, the project does NOT need to put entries in the log. this.syncdb.once("change", () => - this.redux.getProjectActions(this.project_id).log_opened_time(this.path), + this.redux?.getProjectActions(this.project_id).log_opened_time(this.path), ); // project doesn't care about cursors, but browser clients do: @@ -182,6 +176,25 @@ export class JupyterActions extends JupyterActions0 { } } + initUsageInfo = async () => { + while (this._state != "closed") { + try { + const kernel_usage = await getUsageInfo({ + project_id: this.project_id, + compute_server_id: this.getComputeServerIdSync(), + path: this.path, + }); + if (this._state == ("closed" as any)) return; + this.setState({ kernel_usage }); + } catch (err) { + console.log(`WARNING: getUsageInfo -- ${err}`); + } + // Backend actually updates state every 2 seconds, but the + // main cost is network traffic. + await delay(3000); + } + }; + public run_cell( id: string, save: boolean = true, @@ -301,17 +314,8 @@ export class JupyterActions extends JupyterActions0 { await this.format_cells(this.store.get_cell_ids_list(), sync); } - private usage_info_handler(usage: ImmutableUsageInfo): void { - // console.log("jupyter usage", this.path, "→", usage?.toJS()); - this.setState({ kernel_usage: usage }); - } - public async close(): Promise { if (this.is_closed()) return; - if (this.usage_info != null) { - const key = this.usage_info.event_key(this.path); - this.usage_info.off(key, this.usage_info_handler); - } await super.close(); } diff --git a/src/packages/frontend/jupyter/kernelspecs.ts b/src/packages/frontend/jupyter/kernelspecs.ts index ef9be9d2c5..596e55b57a 100644 --- a/src/packages/frontend/jupyter/kernelspecs.ts +++ b/src/packages/frontend/jupyter/kernelspecs.ts @@ -33,7 +33,7 @@ const getKernelSpec = reuseInFlight( return spec; } } - const api = webapp_client.nats_client.projectApi({ + const api = webapp_client.conat_client.projectApi({ project_id, compute_server_id, timeout: 7500, diff --git a/src/packages/frontend/jupyter/logo.tsx b/src/packages/frontend/jupyter/logo.tsx index 9a299d35e2..fe26b5f15e 100644 --- a/src/packages/frontend/jupyter/logo.tsx +++ b/src/packages/frontend/jupyter/logo.tsx @@ -112,7 +112,7 @@ async function getLogo({ if (!noCache && cache[key]) { return cache[key]; } - const api = client.nats_client.projectApi({ project_id }); + const api = client.conat_client.projectApi({ project_id }); const { filename, base64 } = await api.editor.jupyterKernelLogo(kernel, { noCache, }); diff --git a/src/packages/frontend/jupyter/output-messages/__test__/javascript.test.tsx b/src/packages/frontend/jupyter/output-messages/__test__/javascript.test.tsx deleted file mode 100644 index dd77b037bb..0000000000 --- a/src/packages/frontend/jupyter/output-messages/__test__/javascript.test.tsx +++ /dev/null @@ -1,20 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -import React from "react"; -import { shallow } from "enzyme"; -import { Javascript } from "../javascript"; - -describe("test a single string of javascript", () => { - const wrapper = shallow(); - - it("checks the html (a div)", () => { - expect(wrapper.html()).toBe("
    "); - }); - - it("checks the javascript eval side effect", () => { - expect((window as any).value).toBe(5); - }); -}); diff --git a/src/packages/frontend/landing-page/sign-in-hooks.ts b/src/packages/frontend/landing-page/sign-in-hooks.ts index e8f9440f44..3dad1d026e 100644 --- a/src/packages/frontend/landing-page/sign-in-hooks.ts +++ b/src/packages/frontend/landing-page/sign-in-hooks.ts @@ -41,7 +41,7 @@ async function analytics_send(mesg: SignedIn): Promise { }) // .then(response => console.log("Success:", response)) .catch((error) => - console.error("sign-in-hooks::analytics_send error:", error) + console.log("WARNING: sign-in-hooks::analytics_send error:", error), ); } diff --git a/src/packages/frontend/last.ts b/src/packages/frontend/last.ts index dd37884b36..6a4c0db049 100644 --- a/src/packages/frontend/last.ts +++ b/src/packages/frontend/last.ts @@ -24,14 +24,14 @@ export function init() { } }); - if (webapp_client.hub_client.is_connected()) { + if (webapp_client.conat_client.is_connected()) { // These events below currently (due to not having finished the react rewrite) // have to be emited after the page loads, but may happen before. webapp_client.emit("connected"); - if (webapp_client.hub_client.is_signed_in()) { + if (webapp_client.conat_client.is_signed_in()) { webapp_client.emit( "signed_in", - webapp_client.hub_client.get_signed_in_mesg(), + webapp_client.conat_client.signedInMessage, ); } } diff --git a/src/packages/frontend/launch/actions.ts b/src/packages/frontend/launch/actions.ts index 908fba6fde..5a1e48c6d9 100644 --- a/src/packages/frontend/launch/actions.ts +++ b/src/packages/frontend/launch/actions.ts @@ -24,7 +24,6 @@ import { Actions, Store, redux } from "@cocalc/frontend/app-framework"; import * as LS from "@cocalc/frontend/misc/local-storage-typed"; import { QueryParams } from "@cocalc/frontend/misc/query-params"; import { CSILauncher } from "./custom-image"; -import { ShareLauncher } from "./share"; export const NAME = "launch-actions"; const LS_KEY = NAME; @@ -117,8 +116,7 @@ export async function launch() { new CSILauncher(image_id).launch(); return; case "share": - new ShareLauncher(launch).launch(); - return; + throw Error("share launcher is deprecated"); default: console.warn(`launch type "${type}" unknown`); return; diff --git a/src/packages/frontend/launch/share.ts b/src/packages/frontend/launch/share.ts deleted file mode 100644 index 89da95eab5..0000000000 --- a/src/packages/frontend/launch/share.ts +++ /dev/null @@ -1,377 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -/* - With minimal friction copy a share over from some public project - to a project owned by this user, and point the browser at the directory - or file containing the share. - - - if anon user, copy into *the* anon project, and DO not make the default - notebook there. - - - if not anon user: - - if you are a collaborator on the project (or an admin) with this share, - open that project and go to the appropriate path - - if you are not a collaborator, make a new project whose name is maybe - the name or description of the share if possible (or the path); that's - simple and clean. Maybe include info in project description about license, - original URL, etc., or a metadata file with that (which could be used to - improve the directory listing). - - TODO/issues: - - - It's entirely possible that the share is HUGE (e.g., my website is like 60GB, or - maybe an instructors posts a 2GB data file for students), so we don't want to - just naively copy some massive amount of files. I'm not sure how to prevent this - for #v0 though. - - - We do NOT just use the "copy between projects" api, because that requires starting - up the project that is the source of the shared files. We use the public option - below to avoid this. - - - What if a file depends on some other files. Then the directory has to get copied to - get those dependent files, which is a little confusing. - - I'm not making the above blockers for this, because they have been problems exactly as - is for years now, and the current UI requires users to manually do very hard stuff, - which I doubt anybody ever does... -*/ - -/* The launch string should be of the form: - "launch/[shared_id]/path/to/doc/in/a/share" -*/ - -import { alert_message } from "@cocalc/frontend/alerts"; -import { redux } from "@cocalc/frontend/app-framework"; -import { ANON_PROJECT_TITLE } from "@cocalc/frontend/client/anonymous-setup"; -import { is_custom_image } from "@cocalc/frontend/custom-software/util"; -import { query } from "@cocalc/frontend/frame-editors/generic/client"; -import { CSILauncher } from "@cocalc/frontend/launch/custom-image"; -import { webapp_client } from "@cocalc/frontend/webapp-client"; -import { callback2, once, retry_until_success } from "@cocalc/util/async-utils"; -import { FALLBACK_COMPUTE_IMAGE } from "@cocalc/util/db-schema"; -import { len, uuid } from "@cocalc/util/misc"; - -type Relationship = - | "collaborator" // user is a collaborator on the shared project (so just directly open the shared project) - | "fork" // user is a normal user who needs to make a fork of the shared files in a new project (a fork) - | "anonymous"; // user is anonymous, so make a copy of the shared files in their own project - -interface ShareInfo { - id: string; - project_id: string; - path: string; - description?: string; - license?: string; - compute_image: string; -} - -export class ShareLauncher { - readonly share_id: string; - readonly path: string; - private info: ShareInfo; - - constructor(launch: string) { - const v = launch.split("/"); - this.share_id = v[1]; - this.path = v.slice(2).join("/"); - } - - public async launch() { - alert_message({ - type: "info", - title: "Opening a copy of this shared content in a project...", - timeout: 5, - }); - - const store = redux.getStore("account"); - if (!store.get("is_ready")) { - await once(store, "is_ready"); - } - - // Look up the project_id and path for the share from the database. - const info = (this.info = ( - await query({ - no_post: true, // (ugly) since this call is *right* after making an account, so we need to avoid racing for cookie to be set. - query: { - public_paths_by_id: { - id: this.share_id, - project_id: null, - path: null, - description: null, - license: null, - compute_image: null, - }, - }, - }) - ).query.public_paths_by_id); - - //console.log("info = ", info); - if (info == null) { - throw Error(`there is no public share with id ${this.share_id}`); - } - - // Actual path is in the URL and can be much more refined than the share path. - info.path = this.path; - if (info.path.endsWith("/")) { - info.path = info.path.slice(0, info.path.length - 1); - } - - // the compute image's fallback value is "default" (from the time before this field existed) - // don't change it to DEFAULT_COMPUTE_IMAGE - info.compute_image = info.compute_image ?? FALLBACK_COMPUTE_IMAGE; - - // What is our relationship to this public_path? - const relationship: Relationship = await this.get_relationship_to_share( - info.project_id, - ); - - //console.log("relationship = ", relationship); - - switch (relationship) { - case "collaborator": - await this.open_share_as_collaborator(); - alert_message({ - type: "info", - title: "Opened project with the shared content.", - message: - "Since your account already has edit access to this shared content, it has been opened for you.", - block: true, - }); - break; - case "anonymous": - await this.open_share_in_the_anonymous_project(); - alert_message({ - type: "info", - title: `Shared content opened - ${info.description}`, - message: - "You can edit and run this share! Create an account in order to save your changes, collaborate with other people (and much more!).", - block: true, - }); - break; - case "fork": - await this.open_share_in_a_new_project(); - alert_message({ - type: "info", - title: `Shared content opened in a project - ${info.description}`, - message: - "You can edit and run this share in this project. You may want to upgrade this project or copy files to another one of your projects.", - block: true, - }); - break; - default: - throw Error(`unknown relationship "${relationship}"`); - } - - // TODO -- maybe -- write some sort of metadata or a markdown file (e.g., source.md) - // somewhere explaining where this shared file came from (share link, description, etc.). - } - - private async get_relationship_to_share( - project_id: string, - ): Promise { - const account_store = redux.getStore("account"); - if (account_store == null) { - throw Error("account_store MUST be defined"); - } - if (!account_store.get("is_logged_in")) { - throw Error( - "user must be signed in before share launch action is performed", - ); - } - if (account_store.get("is_anonymous")) { - return "anonymous"; - } - if (account_store.get("is_admin")) { - return "collaborator"; // admin is basically viewed as collab on everything for permissions. - } - // OK, now we have a normal non-anonymous non-admin user that is signed in. - // Decide if this is a project they are a collab on or not. - // We do this robustly by querying the projects table for this one project; - // if we are on this project, we'll get a result back, and if not an empty - // object back (since it is outside of our "universe"). Also, we include - // last_active in the query, since otherwise the query always just comes back - // empty as a sort of no-op (probably an edge case bug). - try { - const project = ( - await query({ query: { projects: { project_id, last_active: null } } }) - ).query.projects; - return project == null || len(project) == 0 ? "fork" : "collaborator"; - } catch (err) { - // For non admin get an err when trying to get info about a project that - // we don't have access to. - return "fork"; - } - } - - // Easy: just open it and done! - private open_share_as_collaborator(): void { - const { project_id, path } = this.info; - const target = "files/" + path; - redux.getActions("projects").open_project({ - project_id, - switch_to: true, - target, - }); - } - - private async is_valid_comp_img(img: string): Promise { - const cs = redux.getStore("customize"); - await cs.until_configured(); - const envs = cs.getIn(["software", "environments"]); - return !!envs.get(img); - } - - private async create_target_project({ - title, - compute_image, - }): Promise { - try { - // check, if this is a custom software image and use specific setup code - if (is_custom_image(compute_image)) { - console.log("creating anonymous custom software project"); - // compute_image is like "custom/[image id]/latest" - const image_id = compute_image.split("/")[1]; - const csi = new CSILauncher(image_id); - const project_id = await csi.launch(); - if (project_id == null) { - throw new Error(`image ${compute_image} does not exist`); - } - return project_id; - } else if (await this.is_valid_comp_img(compute_image)) { - // This is one of the standard software images - const actions = redux.getActions("projects"); - console.log("creating anonymous default image project"); - const project_id = await actions.create_project({ - title, - start: true, - description: "", - image: compute_image, - }); - actions.open_project({ project_id, switch_to: true }); - return project_id; - } else { - throw new Error(`unable to handle compute_image='${compute_image}'`); - } - } catch (err) { - throw Error(`unable to create project -- ${err} -- something is wrong`); - } - } - - private async create_and_setup_project(title): Promise { - const { compute_image, project_id, path } = this.info; - const target_project_id = await this.create_target_project({ - title, - compute_image, - }); - - // Change the project title and description to be related to the share, since - // this is very likely the only way it is used (opening this project). - await this.open_share_in_project(project_id, path, target_project_id); - this.set_project_metadata(target_project_id); - } - - private async open_share_in_the_anonymous_project( - max_time_s: number = 30, - ): Promise { - // We wait until the anonymous user exists and then create a project - // (default project creation is intercepted in client/anonymous-setup) - try { - await retry_until_success({ - max_time: max_time_s * 1000, - f: async () => { - const account_store = redux.getStore("account"); - if (account_store == null || !account_store.get("is_anonymous")) { - throw new Error("account does not exist yet ..."); - } - }, - }); - await this.create_and_setup_project(ANON_PROJECT_TITLE); - } catch { - throw Error( - `unable to get anonymous user after waiting ${max_time_s} seconds -- something is wrong`, - ); - } - } - - private async open_share_in_project( - project_id: string, - path: string, - target_project_id: string, - ): Promise { - // Open the project itself. - const projects_actions = redux.getActions("projects"); - projects_actions.open_project({ - project_id: target_project_id, - switch_to: true, - }); - - // Copy the share to the target project. - const actions = redux.getProjectActions(target_project_id); - const id = uuid(); - actions.set_activity({ - id, - status: "Copying shared content to your project...", - }); - - await webapp_client.project_client.copy_path_between_projects({ - public: true, // uses the shared files for the source, NOT the source project! This is very different in KuCalc. - src_project_id: project_id, - src_path: path, - target_project_id, - timeout: 120, - }); - - actions.set_activity({ id, status: "Opening the shared content..." }); - - // Then open the share: - if (actions == null) { - throw Error("target project must exist"); - } - // We have to get the directory listing so we know whether we are opening - // a directory or a file, since unfortunately that info is not part of the share. - const i = path.lastIndexOf("/"); - const containing_path = i == -1 ? "" : path.slice(0, i); - const filename = i == -1 ? path : path.slice(i + 1); - await actions.set_current_path(containing_path); - const store = redux.getProjectStore(target_project_id); - await callback2(store.wait.bind(store), { - until: () => - store.getIn(["directory_listings", 0, containing_path]) != null, - }); - const listing = store.getIn(["directory_listings", 0, containing_path]); - let isdir: boolean = false; - for (const x of listing) { - if (x.get("name") == filename) { - isdir = !!x.get("isdir"); - break; - } - } - if (isdir) { - await actions.set_current_path(path); - } else { - await actions.open_file({ - path, - foreground: true, - foreground_project: true, - }); - } - actions.set_activity({ id, stop: "" }); - } - - private async open_share_in_a_new_project(): Promise { - // Create a new project - await this.create_and_setup_project("Share"); - } - - private set_project_metadata(project_id: string): void { - const { description, path, license } = this.info; - const actions = redux.getActions("projects"); - const title = `Share - ${description ? description : path}`; // lame - const project_description = `${path}\n\n${license}`; - actions.set_project_title(project_id, title); - actions.set_project_description(project_id, project_description); - } -} diff --git a/src/packages/frontend/nats/api/index.ts b/src/packages/frontend/nats/api/index.ts deleted file mode 100644 index d048317314..0000000000 --- a/src/packages/frontend/nats/api/index.ts +++ /dev/null @@ -1,65 +0,0 @@ -/* - -*/ - -import { webapp_client } from "@cocalc/frontend/webapp-client"; -import { type BrowserApi } from "@cocalc/nats/browser-api"; -import { Svcm } from "@nats-io/services"; -import { browserSubject } from "@cocalc/nats/names"; - -export async function initApi() { - const { account_id } = webapp_client; - if (!account_id) { - throw Error("must be signed in"); - } - const { sessionId } = webapp_client.nats_client; - const { jc, nc } = await webapp_client.nats_client.getEnv(); - // @ts-ignore - const svcm = new Svcm(nc); - const subject = browserSubject({ - account_id, - sessionId, - service: "api", - }); - const service = await svcm.add({ - name: `account-${account_id}`, - version: "0.1.0", - description: "CoCalc Web Browser", - }); - const api = service.addEndpoint("api", { subject }); - listen({ api, jc }); -} - -async function listen({ api, jc }) { - for await (const mesg of api) { - const request = jc.decode(mesg.data); - handleApiRequest({ request, mesg, jc }); - } -} - -async function handleApiRequest({ request, mesg, jc }) { - let resp; - try { - const { name, args } = request as any; - console.log("handling browser.api request:", { name }); - resp = (await getResponse({ name, args })) ?? null; - } catch (err) { - resp = { error: `${err}` }; - } - mesg.respond(jc.encode(resp)); -} - -import * as system from "./system"; - -export const browserApi: BrowserApi = { - system, -}; - -async function getResponse({ name, args }) { - const [group, functionName] = name.split("."); - const f = browserApi[group]?.[functionName]; - if (f == null) { - throw Error(`unknown function '${name}'`); - } - return await f(...args); -} diff --git a/src/packages/frontend/nats/api/system.ts b/src/packages/frontend/nats/api/system.ts deleted file mode 100644 index 36e9ccf115..0000000000 --- a/src/packages/frontend/nats/api/system.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { webapp_client } from "@cocalc/frontend/webapp-client"; - -export async function ping() { - return { now: Date.now(), sessionId: webapp_client.nats_client.sessionId }; -} - -import { version as versionNumber } from "@cocalc/util/smc-version"; -export async function version() { - return versionNumber; -} diff --git a/src/packages/frontend/nats/client.ts b/src/packages/frontend/nats/client.ts deleted file mode 100644 index feb3565ad4..0000000000 --- a/src/packages/frontend/nats/client.ts +++ /dev/null @@ -1,688 +0,0 @@ -import * as nats from "nats.ws"; -import { connect, type CoCalcNatsConnection } from "./connection"; -import { redux } from "@cocalc/frontend/app-framework"; -import type { WebappClient } from "@cocalc/frontend/client/client"; -import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; -import * as jetstream from "@nats-io/jetstream"; -import { - createSyncTable, - type NatsSyncTable, - NatsSyncTableFunction, -} from "@cocalc/nats/sync/synctable"; -import { randomId } from "@cocalc/nats/names"; -import { browserSubject, projectSubject } from "@cocalc/nats/names"; -import { parse_query } from "@cocalc/sync/table/util"; -import { sha1 } from "@cocalc/util/misc"; -import { keys } from "lodash"; -import { type HubApi, initHubApi } from "@cocalc/nats/hub-api"; -import { type ProjectApi, initProjectApi } from "@cocalc/nats/project-api"; -import { type BrowserApi, initBrowserApi } from "@cocalc/nats/browser-api"; -import { getPrimusConnection } from "@cocalc/nats/primus"; -import { isValidUUID } from "@cocalc/util/misc"; -import { createOpenFiles, OpenFiles } from "@cocalc/nats/sync/open-files"; -import { PubSub } from "@cocalc/nats/sync/pubsub"; -import type { ChatOptions } from "@cocalc/util/types/llm"; -import { kv, type KVOptions, type KV } from "@cocalc/nats/sync/kv"; -import { dkv, type DKVOptions, type DKV } from "@cocalc/nats/sync/dkv"; -import { dko, type DKO } from "@cocalc/nats/sync/dko"; -import { - stream, - type UserStreamOptions, - type Stream, -} from "@cocalc/nats/sync/stream"; -import { dstream } from "@cocalc/nats/sync/dstream"; -import { initApi } from "@cocalc/frontend/nats/api"; -import { delay } from "awaiting"; -import { callNatsService, createNatsService } from "@cocalc/nats/service"; -import type { - CallNatsServiceFunction, - CreateNatsServiceFunction, -} from "@cocalc/nats/service"; -import { listingsClient } from "@cocalc/nats/service/listings"; -import { - computeServerManager, - type Options as ComputeServerManagerOptions, -} from "@cocalc/nats/compute/manager"; -import getTime, { getSkew, init as initTime } from "@cocalc/nats/time"; -import { llm } from "@cocalc/nats/llm/client"; -import { inventory } from "@cocalc/nats/sync/inventory"; -import { EventEmitter } from "events"; -import { - getClient as getClientWithState, - setNatsClient, - type ClientWithState, - getEnv, -} from "@cocalc/nats/client"; -import type { ConnectionInfo } from "./types"; -import { fromJS } from "immutable"; -import { requestMany } from "@cocalc/nats/service/many"; -import Cookies from "js-cookie"; -import { ACCOUNT_ID_COOKIE } from "@cocalc/frontend/client/client"; -import { isConnected, waitUntilConnected } from "@cocalc/nats/util"; -import { info as refCacheInfo } from "@cocalc/util/refcache"; -import * as tieredStorage from "@cocalc/nats/tiered-storage/client"; - -const NATS_STATS_INTERVAL = 2500; - -const DEFAULT_TIMEOUT = 15000; - -declare var DEBUG: boolean; - -export class NatsClient extends EventEmitter { - client: WebappClient; - private sc = nats.StringCodec(); - private jc = nats.JSONCodec(); - private nc?: CoCalcNatsConnection; - public nats = nats; - public jetstream = jetstream; - public hub: HubApi; - public sessionId = randomId(); - private openFilesCache: { [project_id: string]: OpenFiles } = {}; - private clientWithState: ClientWithState; - - constructor(client: WebappClient) { - super(); - this.setMaxListeners(100); - this.client = client; - this.hub = initHubApi(this.callHub); - this.initBrowserApi(); - this.initNatsClient(); - this.on("state", (state) => { - this.emit(state); - this.setConnectionState(state); - }); - } - - private initNatsClient = async () => { - let d = 100; - // wait until you're signed in -- usually the account_id cookie ensures this, - // but if somehow it got deleted, the normal websocket sign in message from the - // hub also provides the account_id right now. That will eventually go away, - // at which point this should become fatal. - if (!this.client.account_id) { - while (!this.client.account_id) { - await delay(d); - d = Math.min(3000, d * 1.3); - } - // we know the account_id, so set it so next time sign is faster. - Cookies.set(ACCOUNT_ID_COOKIE, this.client.account_id); - } - setNatsClient({ - account_id: this.client.account_id, - getNatsEnv: this.getNatsEnv, - reconnect: this.reconnect, - getLogger: DEBUG - ? (name) => { - return { - info: (...args) => console.info(name, ...args), - debug: (...args) => console.log(name, ...args), - warn: (...args) => console.warn(name, ...args), - }; - } - : undefined, - }); - this.clientWithState = getClientWithState(); - this.clientWithState.on("state", (state) => { - if (state != "closed") { - console.log("NATS: ", state); - this.emit(state); - } - }); - initTime(); - }; - - getEnv = async () => await getEnv(); - - private initBrowserApi = async () => { - if (!this.client.account_id) { - // it's impossible to initialize the browser api if user is not signed in, - // and there is no way to ever sign in without explicitly leaving this - // page and coming back. - return; - } - // have to delay so that this.client is fully created. - await delay(1); - let d = 2000; - while (true) { - try { - await initApi(); - return; - } catch (err) { - console.log( - `WARNING: failed to initialize browser api (will retry) -- ${err}`, - ); - } - d = Math.min(25000, d * 1.3) + Math.random(); - await delay(d); - } - }; - - private getConnection = reuseInFlight(async () => { - if (this.nc != null) { - return this.nc; - } - this.nc = await connect(); - this.setConnectionState("connected"); - this.monitorConnectionState(this.nc); - this.reportConnectionStats(this.nc); - return this.nc; - }); - - reconnect = reuseInFlight(async () => { - if (this.nc != null) { - console.log("NATS connection: reconnecting..."); - this.standby(); - await delay(50); - await this.resume(); - } - }); - - // if there is a connection, put it in standby - standby = () => { - this.nc?.standby(); - }; - // if there is a connection, resume it - resume = async () => { - await this.nc?.resume(); - }; - - // reconnect to nats with access to additional projects. - // If you request projects that you're not actually a collaborator - // on, then it will silently NOT give you permission to use them. - addProjectPermissions = async (project_ids: string[]) => { - if (this.nc == null) { - throw Error("must have a connection"); - } - await this.nc.addProjectPermissions(project_ids); - }; - - private setConnectionState = (state?) => { - const page = redux?.getActions("page"); - if (page == null) { - return; - } - page.setState({ - nats: { - state: state ?? this.clientWithState.state, - data: this.nc?.stats(), - }, - } as any); - }; - - private monitorConnectionState = async (nc) => { - for await (const _ of nc.statusOfCurrentConnection()) { - this.setConnectionState(); - } - }; - - private reportConnectionStats = async (nc) => { - while (true) { - const store = redux?.getStore("page"); - const actions = redux?.getActions("page"); - if (store != null && actions != null) { - const cur = store.get("nats") ?? (fromJS({}) as any); - const nats = cur.set("data", fromJS(nc.stats())); - if (!cur.equals(nats)) { - actions.setState({ nats }); - } - } - await delay(NATS_STATS_INTERVAL); - } - }; - - callNatsService: CallNatsServiceFunction = async (options) => { - return await callNatsService(options); - }; - - createNatsService: CreateNatsServiceFunction = (options) => { - return createNatsService(options); - }; - - // TODO: plan to deprecated... - projectWebsocketApi = async ({ - project_id, - mesg, - timeout = DEFAULT_TIMEOUT, - }) => { - const { nc } = await this.getEnv(); - const subject = projectSubject({ project_id, service: "browser-api" }); - const resp = await nc.request(subject, this.jc.encode(mesg), { - timeout, - }); - return this.jc.decode(resp.data); - }; - - private callHub = async ({ - service = "api", - name, - args = [], - timeout = DEFAULT_TIMEOUT, - requestMany: requestMany0 = false, - }: { - service?: string; - name: string; - args: any[]; - timeout?: number; - // requestMany -- if true do a requestMany request, which is more complicated/slower, but - // supports arbitrarily large responses irregardless of the nats server max message size. - requestMany?: boolean; - }) => { - const { nc } = await this.getEnv(); - const subject = `hub.account.${this.client.account_id}.${service}`; - try { - const data = this.jc.encode({ - name, - args, - }); - let resp; - await waitUntilConnected(); - if (requestMany0) { - resp = await requestMany({ nc, subject, data, maxWait: timeout }); - } else { - resp = await nc.request(subject, data, { timeout }); - } - return this.jc.decode(resp.data); - } catch (err) { - err.message = `${err.message} - callHub: subject='${subject}', name='${name}', `; - throw err; - } - }; - - // Returns api for RPC calls to the project with typescript support! - // if compute_server_id is NOT given then: - // if path is given use compute server id for path (assuming mapping is loaded) - // if path is not given, use current project default - projectApi = ({ - project_id, - compute_server_id, - path, - timeout = DEFAULT_TIMEOUT, - }: { - project_id: string; - path?: string; - compute_server_id?: number; - // IMPORTANT: this timeout is only AFTER user is connected. - timeout?: number; - }): ProjectApi => { - if (!isValidUUID(project_id)) { - throw Error(`project_id = '${project_id}' must be a valid uuid`); - } - let lastAddedPermission = 0; - if (compute_server_id == null) { - const actions = redux.getProjectActions(project_id); - if (path != null) { - compute_server_id = - actions.getComputeServerIdForFile({ path }) ?? - actions.getComputeServerId(); - } else { - compute_server_id = actions.getComputeServerId(); - } - } - const callProjectApi = async ({ name, args }) => { - const opts = { - project_id, - compute_server_id, - timeout, - service: "api", - name, - args, - }; - try { - await waitUntilConnected(); - return await this.callProject(opts); - } catch (err) { - if ( - err.code == "PERMISSIONS_VIOLATION" && - Date.now() - lastAddedPermission >= 30000 - ) { - lastAddedPermission = Date.now(); - await this.addProjectPermissions([project_id]); - await waitUntilConnected(); - return await this.callProject(opts); - } else { - throw err; - } - } - }; - return initProjectApi(callProjectApi); - }; - - private callProject = async ({ - service = "api", - project_id, - compute_server_id, - name, - args = [], - timeout = DEFAULT_TIMEOUT, - }: { - service?: string; - project_id: string; - compute_server_id?: number; - name: string; - args: any[]; - timeout?: number; - }) => { - const { nc } = await this.getEnv(); - const subject = projectSubject({ project_id, compute_server_id, service }); - const mesg = this.jc.encode({ - name, - args, - }); - let resp; - try { - await waitUntilConnected(); - resp = await nc.request(subject, mesg, { timeout }); - } catch (err) { - if (err.code == "PERMISSIONS_VIOLATION") { - // request update of our credentials to include this project, then try again - await (nc as any).addProjectPermissions([project_id]); - await waitUntilConnected(); - resp = await nc.request(subject, mesg, { timeout }); - } else { - throw err; - } - } - return this.jc.decode(resp.data); - }; - - private callBrowser = async ({ - service = "api", - sessionId, - name, - args = [], - timeout = DEFAULT_TIMEOUT, - }: { - service?: string; - sessionId: string; - name: string; - args: any[]; - timeout?: number; - }) => { - const { nc } = await this.getEnv(); - const subject = browserSubject({ - account_id: this.client.account_id, - sessionId, - service, - }); - const mesg = this.jc.encode({ - name, - args, - }); - // console.log("request to subject", { subject, name, args }); - await waitUntilConnected(); - const resp = await nc.request(subject, mesg, { timeout }); - return this.jc.decode(resp.data); - }; - - browserApi = ({ - sessionId, - timeout = DEFAULT_TIMEOUT, - }: { - sessionId: string; - timeout?: number; - }): BrowserApi => { - const callBrowserApi = async ({ name, args }) => { - return await this.callBrowser({ - sessionId, - timeout, - service: "api", - name, - args, - }); - }; - return initBrowserApi(callBrowserApi); - }; - - request = async (subject: string, data: string) => { - const { nc } = await this.getEnv(); - await waitUntilConnected(); - const resp = await nc.request(subject, this.sc.encode(data)); - return this.sc.decode(resp.data); - }; - - consumer = async (stream: string) => { - const { nc } = await this.getEnv(); - const js = jetstream.jetstream(nc); - return await js.consumers.get(stream); - }; - - private getNatsEnv = async () => { - return { - sha1, - jc: this.jc, - nc: await this.getConnection(), - }; - }; - - synctable: NatsSyncTableFunction = async ( - query, - options?, - ): Promise => { - query = parse_query(query); - const table = keys(query)[0]; - const obj = options?.obj; - if (obj != null) { - for (const k in obj) { - query[table][0][k] = obj[k]; - } - } - if (options?.project_id != null && query[table][0]["project_id"] === null) { - query[table][0]["project_id"] = options.project_id; - } - const s = createSyncTable({ - ...options, - query, - env: await this.getEnv(), - account_id: this.client.account_id, - }); - await s.init(); - return s; - }; - - changefeedInterest = async (query, noError?: boolean) => { - // express interest - // (re-)start changefeed going - try { - await this.client.nats_client.callHub({ - service: "db", - name: "userQuery", - args: [{ changes: true, query }], - }); - } catch (err) { - if (noError) { - console.log(`WARNING: changefeed -- ${err}`, query); - return; - } else { - throw err; - } - } - }; - - changefeed = async (query, options?) => { - this.changefeedInterest(query, true); - return await this.synctable(query, options); - }; - - // DEPRECATED - primus = async (project_id: string) => { - return getPrimusConnection({ - subject: projectSubject({ - project_id, - compute_server_id: 0, - service: "primus", - }), - env: await this.getEnv(), - role: "client", - id: this.sessionId, - }); - }; - - openFiles = reuseInFlight(async (project_id: string) => { - if (this.openFilesCache[project_id] == null) { - const openFiles = await createOpenFiles({ - project_id, - }); - this.openFilesCache[project_id] = openFiles; - openFiles.on("closed", () => { - delete this.openFilesCache[project_id]; - }); - openFiles.on("change", (entry) => { - if (entry.deleted?.deleted) { - setDeleted({ - project_id, - path: entry.path, - deleted: entry.deleted.time, - }); - } else { - setNotDeleted({ project_id, path: entry.path }); - } - }); - const recentlyDeletedPaths: any = {}; - for (const { path, deleted } of openFiles.getAll()) { - if (deleted?.deleted) { - recentlyDeletedPaths[path] = deleted.time; - } - } - const store = redux.getProjectStore(project_id); - store.setState({ recentlyDeletedPaths }); - } - return this.openFilesCache[project_id]!; - }); - - closeOpenFiles = (project_id) => { - this.openFilesCache[project_id]?.close(); - }; - - pubsub = async ({ - project_id, - path, - name, - }: { - project_id: string; - path?: string; - name: string; - }) => { - return new PubSub({ project_id, path, name, env: await this.getEnv() }); - }; - - // Evaluate an llm. This streams the result if stream is given an option, - // AND it also always returns the result. - llm = async (opts: ChatOptions): Promise => { - return await llm({ account_id: this.client.account_id, ...opts }); - }; - - stream = async ( - opts: Partial & { name: string }, - ): Promise> => { - if (!opts.account_id && !opts.project_id && opts.limits != null) { - throw Error("account client can't set limits on public stream"); - } - return await stream({ env: await this.getEnv(), ...opts }); - }; - - dstream = dstream; - - kv = async ( - opts: Partial & { name: string }, - ): Promise> => { - // if (!opts.account_id && !opts.project_id && opts.limits != null) { - // throw Error("account client can't set limits on public stream"); - // } - return await kv({ env: await this.getEnv(), ...opts }); - }; - - dkv = async ( - opts: Partial & { name: string }, - ): Promise> => { - // if (!opts.account_id && !opts.project_id && opts.limits != null) { - // throw Error("account client can't set limits on public stream"); - // } - return await dkv({ env: await this.getEnv(), ...opts }); - }; - - dko = async ( - opts: Partial & { name: string }, - ): Promise> => { - // if (!opts.account_id && !opts.project_id && opts.limits != null) { - // throw Error("account client can't set limits on public stream"); - // } - return await dko({ env: await this.getEnv(), ...opts }); - }; - - listings = async (opts: { - project_id: string; - compute_server_id?: number; - }) => { - return await listingsClient(opts); - }; - - computeServerManager = async (options: ComputeServerManagerOptions) => { - const f = async () => { - const M = computeServerManager(options); - await M.init(); - return M; - }; - try { - return await f(); - } catch (err) { - if (err.code == "PERMISSIONS_VIOLATION" && options.project_id) { - await this.addProjectPermissions([options.project_id]); - return await f(); - } else { - throw err; - } - } - }; - - getTime = (): number => { - return getTime(); - }; - - getSkew = async (): Promise => { - return await getSkew(); - }; - - inventory = async (location: { - account_id?: string; - project_id?: string; - }) => { - const inv = await inventory(location); - // @ts-ignore - if (console.log_original != null) { - const ls_orig = inv.ls; - // @ts-ignore - inv.ls = (opts) => ls_orig({ ...opts, log: console.log_original }); - } - return inv; - }; - - info = async (nc): Promise => { - // info about a nats connection - return this.jc.decode( - (await nc.request("$SYS.REQ.USER.INFO")).data, - ) as ConnectionInfo; - }; - - isConnected = async () => await isConnected(); - waitUntilConnected = async () => await waitUntilConnected(); - - refCacheInfo = () => refCacheInfo(); - - tieredStorage = tieredStorage; -} - -function setDeleted({ project_id, path, deleted }) { - if (!redux.hasProjectStore(project_id)) { - return; - } - const actions = redux.getProjectActions(project_id); - actions.setRecentlyDeleted(path, deleted); -} - -function setNotDeleted({ project_id, path }) { - if (!redux.hasProjectStore(project_id)) { - return; - } - const actions = redux.getProjectActions(project_id); - actions.setRecentlyDeleted(path, 0); -} diff --git a/src/packages/frontend/nats/connection.ts b/src/packages/frontend/nats/connection.ts deleted file mode 100644 index c48234e39e..0000000000 --- a/src/packages/frontend/nats/connection.ts +++ /dev/null @@ -1,498 +0,0 @@ -/* -This should work for clients just like a normal NATS connection, but it -also dynamically reconnects to adjust permissions for projects -a browser client may connect to. - -This is needed ONLY because: - - - in NATS you can't change the permissions of an existing - connection when auth is done via auth-callout like we're doing. - This could become possible in the future, with some change - to the NATS server. Or maybe I just don't understand it. - - - There is a relatively small limit on the number of permissions for - one connection, which must be explicitly listed on creation of - the connection. However, in CoCalc, a single account can be a - collaborator on 20,000+ projects, and connect to any one of them - at any time. - - -The other option would be to have a separate nats connection for each -project that the browser has open. This is also viable and probably -simpler. We basically do that with primus. The drawbacks: - - - browsers limit the number of websockets for a tab to about 200 - - more connections ==> more load on nats and limits scalability - -I generally "feel" like this should be the optimal approach given -all the annoying constraints. We will likely do something -involving always including recent projects. - ---- - -Subscription Leaks: - -This code in a browser is useful for monitoring the number of subscriptions: - -setInterval(()=>console.log(cc.redux.getStore('page').get('nats').toJS().data.numSubscriptions),1000) - -If things are off, look at - -cc.client.nats_client.refCacheInfo() -*/ - -import { appBasePath } from "@cocalc/frontend/customize/app-base-path"; -import { webapp_client } from "@cocalc/frontend/webapp-client"; -import { join } from "path"; -import type { - NatsConnection, - ServerInfo, - Payload, - PublishOptions, - RequestOptions, - Msg, - SubscriptionOptions, - RequestManyOptions, - Stats, - Status, - Subscription, -} from "@nats-io/nats-core"; -import { connect as natsConnect } from "nats.ws"; -import { inboxPrefix } from "@cocalc/nats/names"; -import { CONNECT_OPTIONS } from "@cocalc/util/nats"; -import { EventEmitter } from "events"; -import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; -import { asyncDebounce } from "@cocalc/util/async-utils"; -import { delay } from "awaiting"; -import { - getPermissionsCache, - type NatsProjectPermissionsCache, -} from "./permissions-cache"; -import { isEqual } from "lodash"; -import { alert_message } from "@cocalc/frontend/alerts"; -import jsonStable from "json-stable-stringify"; - -const MAX_SUBSCRIPTIONS = 400; - -// When we create a new connection to change permissions (i.e., open a project -// we have not opened in a while), we wait this long before draining the -// old connection. Draining immediately should work fine and be more efficient; -// however, it might cause more "disruption". On the other hand, this might -// mask a subtle bug hence set this to 0 for some debugging purposes. -const DELAY_UNTIL_DRAIN_PREVIOUS_CONNECTION_MS = 30 * 1000; -// for debugging/testing -// const DELAY_UNTIL_DRAIN_PREVIOUS_CONNECTION_MS = 0; - -function natsWebsocketUrl() { - return `${location.protocol == "https:" ? "wss" : "ws"}://${location.host}${join(appBasePath, "nats")}`; -} - -function connectingMessage({ server, project_ids }) { - console.log( - `Connecting to ${server} to use ${JSON.stringify(project_ids)}...`, - ); -} - -const getNewNatsConn = reuseInFlight(async ({ cache, user }) => { - const account_id = await getAccountId(); - if (!account_id) { - throw Error("you must be signed in before connecting to NATS"); - } - const server = natsWebsocketUrl(); - const project_ids = cache.get(); - connectingMessage({ server, project_ids }); - const options = { - name: jsonStable(user), - user: `account-${account_id}`, - ...CONNECT_OPTIONS, - servers: [server], - inboxPrefix: inboxPrefix({ account_id }), - }; - while (true) { - try { - console.log("Connecting to NATS..."); - return await natsConnect(options); - } catch (err) { - console.log(`WARNING: failed to connect to NATS -- will retry -- ${err}`); - await delay(3000); - } - } -}); - -// This is a hack to get around circular import during initial page load. -// TODO: properly clean up the import order -async function getAccountId() { - try { - return webapp_client.account_id; - } catch { - await delay(1); - return webapp_client.account_id; - } -} - -let cachedConnection: CoCalcNatsConnection | null = null; -export const connect = reuseInFlight(async () => { - if (cachedConnection != null) { - return cachedConnection; - } - const account_id = await getAccountId(); - const cache = getPermissionsCache(); - const project_ids = cache.get(); - const user = { account_id, project_ids }; - const nc = await getNewNatsConn({ cache, user }); - cachedConnection = new CoCalcNatsConnection(nc, user, cache); - return cachedConnection; -}); - -// There should be at most one single global instance of CoCalcNatsConnection! It -// is responsible for managing any connection to nats. It is assumed that nothing else -// does and that there is only one of these. -class CoCalcNatsConnection extends EventEmitter implements NatsConnection { - conn: NatsConnection; - prev: NatsConnection[] = []; - private standbyMode = false; - info?: ServerInfo; - protocol; - options; - user: { account_id: string; project_ids: string[] }; - permissionsCache: NatsProjectPermissionsCache; - currStatus?; - - constructor(conn, user, permissionsCache) { - super(); - this.setMaxListeners(500); - this.conn = conn; - this.protocol = conn.protocol; - this.info = conn.info; - this.options = conn.options; - this.user = { - project_ids: uniq(user.project_ids), - account_id: user.account_id, - }; - this.permissionsCache = permissionsCache; - this.updateCache(); - } - - standby = () => { - if (this.standbyMode) { - return; - } - this.standbyMode = true; - // standby is used when you are idle, so you should have nothing important to save. - // Also, we can't get rid of this.conn until we have a new connection, which would make - // no sense here.... so we do NOT use this.conn.drain(). - this.conn.close(); - // @ts-ignore - if (this.conn.protocol) { - // @ts-ignore - this.conn.protocol.connected = false; - } - }; - - resume = async () => { - console.log("nats connection: resume"); - if (!this.standbyMode) { - console.log("nats connection: not in standby mode"); - return; - } - this.standbyMode = false; - // @ts-ignore - if (this.conn.protocol?.connected) { - console.log("nats connection: already connected"); - return; - } - console.log("nats connection: getNewNatsConn"); - const conn = await getNewNatsConn({ - cache: this.permissionsCache, - user: this.user, - }); - console.log("nats connection: got conn"); - // @ts-ignore - this.conn = conn; - // @ts-ignore - this.protocol = conn.protocol; - // @ts-ignore - this.info = conn.info; - // @ts-ignore - this.options = conn.options; - this.emit("reconnect"); - }; - - // gets *actual* projects that this connection has permission to access - getProjectPermissions = async (): Promise => { - const info = await this.getConnectionInfo(); - const project_ids: string[] = []; - for (const x of info.data.permissions.publish.allow) { - if (x.startsWith("project.")) { - const v = x.split("."); - project_ids.push(v[1]); - } - } - return project_ids; - }; - - // one time on first connection we set the cache to match - // the actual projects, so we don't keep requesting ones we - // don't have access to, e.g., on sign out, then sign in as - // different user (or being removed as collaborator). - private updateCache = async () => { - try { - this.permissionsCache.set(await this.getProjectPermissions()); - } catch {} - }; - - getConnectionInfo = async () => { - return await webapp_client.nats_client.info(this.conn); - }; - - private subscriptionPenalty = 20000; - numSubscriptions = () => { - // @ts-ignore - let subs = this.conn.protocol.subscriptions.subs.size; - for (const nc of this.prev) { - // @ts-ignore - subs += nc.protocol.subscriptions.subs.size; - } - if (subs >= MAX_SUBSCRIPTIONS) { - // For now, we put them in standby for a bit - // then resume. This saves any work and disconnects them. - // They then get reconnected. This might help. - console.warn( - `WARNING: Using ${subs} subscriptions which exceeds the limit of ${MAX_SUBSCRIPTIONS}.`, - ); - alert_message({ - type: "warning", - message: - "Your browser is using too many resources; refresh your browser or close some files.", - }); - this.standby(); - this.subscriptionPenalty *= 1.25; - setTimeout(this.resume, this.subscriptionPenalty); - } - return subs; - }; - - getSubscriptions = (): string[] => { - const subjects: string[] = []; - // @ts-ignore - for (const sub of this.conn.protocol.subscriptions.subs) { - subjects.push(sub[1].subject); - } - return subjects; - }; - - addProjectPermissions = async (project_ids: string[]) => { - this.permissionsCache.add(project_ids); - await this.updateProjectPermissions(); - }; - - // this is debounce since adding permissions tends to come in bursts: - private updateProjectPermissions = asyncDebounce( - async () => { - let project_ids = this.permissionsCache.get(); - if (isEqual(this.user.project_ids, project_ids)) { - // nothing to do - return; - } - const account_id = await getAccountId(); - if (!account_id) { - throw Error("you must be signed in before connecting to NATS"); - } - const user = { - account_id, - project_ids, - }; - const server = natsWebsocketUrl(); - connectingMessage({ server, project_ids }); - const options = { - // name: used to convey who we claim to be: - name: jsonStable(user), - // user: displayed in logs - user: `account-${account_id}`, - ...CONNECT_OPTIONS, - servers: [server], - inboxPrefix: inboxPrefix({ account_id }), - }; - const cur = this.conn; - const conn = (await natsConnect(options)) as any; - - this.conn = conn; - this.prev.push(cur); - this.currStatus?.stop(); - - this.protocol = conn.protocol; - this.info = conn.info; - this.options = options; - this.user = user; - // tell clients they should reconnect, since the connection they - // had used is going to drain soon. - this.emit("reconnect"); - // we wait a while, then drain the previous connection. - // Since connection usually change rarely, it's fine to wait a while, - // to minimize disruption. Make this short as a sort of "bug stress test". - delayThenDrain(cur, DELAY_UNTIL_DRAIN_PREVIOUS_CONNECTION_MS); - }, - 1000, - { leading: true, trailing: true }, - ); - - async closed(): Promise { - return await this.conn.closed(); - } - - async close(): Promise { - await this.conn.close(); - } - - publish(subject: string, payload?: Payload, options?: PublishOptions): void { - this.conn.publish(subject, payload, options); - } - - publishMessage(msg: Msg): void { - this.conn.publishMessage(msg); - } - - respondMessage(msg: Msg): boolean { - return this.conn.respondMessage(msg); - } - - subscribe(subject: string, opts?: SubscriptionOptions): Subscription { - return this.conn.subscribe(subject, opts); - } - - // not in the public api, but used by jetstream. - _resub(s: Subscription, subject: string, max?: number) { - return (this.conn as any)._resub(s, subject, max); - } - - // not in the public api - _check(subject: string, sub: boolean, pub: boolean) { - return (this.conn as any)._check(subject, sub, pub); - } - - async request( - subject: string, - payload?: Payload, - opts?: RequestOptions, - ): Promise { - return await this.conn.request(subject, payload, opts); - } - - async requestMany( - subject: string, - payload?: Payload, - opts?: Partial, - ): Promise> { - return await this.conn.requestMany(subject, payload, opts); - } - - async flush(): Promise { - this.conn.flush(); - } - - async drain(): Promise { - await this.conn.drain(); - } - - isClosed(): boolean { - return this.conn.isClosed(); - } - - isDraining(): boolean { - return this.conn.isDraining(); - } - - getServer(): string { - return this.conn.getServer(); - } - - // The kv and stream clients use this, which alerts when connection is closing. - // They also get the 'reconnect' event and drop this connection and get a new one, - // thus also getting a new status. - status(): AsyncIterable { - return this.conn.status(); - } - - // The main client here (./client.ts) uses this to know the status of the primary - // connection, mainly for presentation in the UI. Thus this has to always have - // the latest connection status. - async *statusOfCurrentConnection() { - while (true) { - this.currStatus = this.conn.status(); - for await (const x of this.currStatus) { - this.emit("status", x); - yield x; - } - } - } - - // sum total of all data across *all* connections we've made here. - stats(): Stats & { numSubscriptions: number } { - // @ts-ignore: undocumented API - let { inBytes, inMsgs, outBytes, outMsgs } = this.conn.stats(); - for (const conn of this.prev) { - // @ts-ignore - const x = conn.stats(); - inBytes += x.inBytes; - outBytes += x.outBytes; - inMsgs += x.inMsgs; - outMsgs += x.outMsgs; - } - return { - inBytes, - inMsgs, - outBytes, - outMsgs, - numSubscriptions: this.numSubscriptions(), - }; - } - - async rtt(): Promise { - return await this.conn.rtt(); - } - - async reconnect(): Promise { - try { - await this.conn.reconnect(); - } catch (err) { - console.warn(`NATS reconnect failed -- ${err}`); - } - } - - get features() { - return this.protocol.features; - } - - getServerVersion(): SemVer | undefined { - const info = this.info; - return info ? parseSemVer(info.version) : undefined; - } -} - -async function delayThenDrain(conn, time) { - await delay(time); - try { - await conn.drain(); - } catch (err) { - console.log("delayThenDrain err", err); - } -} - -export { type CoCalcNatsConnection }; - -export type SemVer = { major: number; minor: number; micro: number }; -export function parseSemVer(s = ""): SemVer { - const m = s.match(/(\d+).(\d+).(\d+)/); - if (m) { - return { - major: parseInt(m[1]), - minor: parseInt(m[2]), - micro: parseInt(m[3]), - }; - } - throw new Error(`'${s}' is not a semver value`); -} - -function uniq(v: string[]): string[] { - return Array.from(new Set(v)); -} diff --git a/src/packages/frontend/nats/permissions-cache.ts b/src/packages/frontend/nats/permissions-cache.ts deleted file mode 100644 index 21d9393fb7..0000000000 --- a/src/packages/frontend/nats/permissions-cache.ts +++ /dev/null @@ -1,124 +0,0 @@ -import { isValidUUID } from "@cocalc/util/misc"; -import { appBasePath } from "@cocalc/frontend/customize/app-base-path"; - -// This limit is because there is a limit on -// the length authentication protocol message, which is what we use to send the list of projects. -// This limit is the max_control_line (see https://docs.nats.io/running-a-nats-service/configuration) -// By default it is 4KB, which supports about 50 projects. We increase it in the server -// to 32KB and allow up to 250 projects, which is way more than enough (oldest projects are -// automatically removed as needed). -const MAX_PROJECT_PERMISSIONS = 250; -const NORMAL_PROJECT_PERMISSIONS = 25; -const CUTOFF = 1000 * 60 * 60 * 24 * 7; // 1 week ago - -// For dev/testing -- uncomment these to cause chaos as you click to open projects -// and close them and if you open several at once there's no permissions. then -// test that things don't crash, but just keep trying, properly. -// const MAX_PROJECT_PERMISSIONS = 4; -// const NORMAL_PROJECT_PERMISSIONS = 1; -// const CUTOFF = 1000 * 30; - -type NatsProjectCache = { [project_id: string]: number }; - -const localStorageKey = `${appBasePath}-nats-projects`; -console.log(localStorageKey); - -let cache: NatsProjectPermissionsCache | null = null; -export function getPermissionsCache() { - if (cache == null) { - cache = new NatsProjectPermissionsCache(); - } - return cache; -} - -export class NatsProjectPermissionsCache { - cache: NatsProjectCache; - - constructor() { - this.cache = this.loadCache(); - } - - add = (project_ids: string[]) => { - for (const project_id of project_ids) { - if (!isValidUUID(project_id)) { - throw Error(`invalid project_id -- ${project_id}`); - } - this.cache[project_id] = Date.now(); - } - this.enforceLimits(); - this.saveCache(); - }; - - get = () => { - return Object.keys(this.cache).sort(); - }; - - set = (project_ids: string[]) => { - this.cache = {}; - const now = Date.now(); - for (const project_id of project_ids) { - this.cache[project_id] = now; - } - this.enforceLimits(); - this.saveCache(); - }; - - private enforceLimits = () => { - const k = Object.keys(this.cache); - if (k.length <= NORMAL_PROJECT_PERMISSIONS) { - return; - } - let n = k.length; - const cutoff = new Date(Date.now() - CUTOFF).valueOf(); - for (const project_id in this.cache) { - if (this.cache[project_id] <= cutoff) { - delete this.cache[project_id]; - n -= 1; - if (n <= NORMAL_PROJECT_PERMISSIONS) { - return; - } - } - } - if (n > MAX_PROJECT_PERMISSIONS) { - const v = Object.values(this.cache); - v.sort(); - const c = v.slice(-MAX_PROJECT_PERMISSIONS)[0]; - if (c != null) { - for (const project_id in this.cache) { - if (this.cache[project_id] <= c) { - delete this.cache[project_id]; - n -= 1; - if (n <= MAX_PROJECT_PERMISSIONS) { - return; - } - } - } - } - } - }; - - private saveCache = () => { - localStorage[localStorageKey] = JSON.stringify(this.cache); - }; - - private loadCache = (): NatsProjectCache => { - const s = localStorage[localStorageKey]; - if (!s) { - return {}; - } - // don't trust s at all; - try { - const a = JSON.parse(s) as any; - const cache: NatsProjectCache = {}; - for (const project_id in a) { - if (isValidUUID(project_id)) { - cache[project_id] = parseInt(a[project_id]); - } - } - return cache; - } catch (err) { - console.log("warning: ", err); - return {}; - } - }; -} diff --git a/src/packages/frontend/package.json b/src/packages/frontend/package.json index d5c3c4b420..18ef8d777f 100644 --- a/src/packages/frontend/package.json +++ b/src/packages/frontend/package.json @@ -41,10 +41,10 @@ "@cocalc/assets": "workspace:*", "@cocalc/cdn": "workspace:*", "@cocalc/comm": "workspace:*", + "@cocalc/conat": "workspace:*", "@cocalc/frontend": "workspace:*", "@cocalc/jupyter": "workspace:*", "@cocalc/local-storage-lru": "^2.4.3", - "@cocalc/nats": "workspace:*", "@cocalc/sync": "workspace:*", "@cocalc/util": "workspace:*", "@cocalc/widgets": "^1.2.0", @@ -52,25 +52,23 @@ "@dnd-kit/modifiers": "^6.0.1", "@dnd-kit/sortable": "^7.0.2", "@dnd-kit/utilities": "^3.2.1", - "@isaacs/ttlcache": "^1.2.1", + "@isaacs/ttlcache": "^1.4.1", "@jupyter-widgets/base": "^4.1.1", "@jupyter-widgets/controls": "5.0.0-rc.2", "@jupyter-widgets/output": "^4.1.0", - "@lumino/widgets": "^1.31.1", "@microlink/react-json-view": "^1.23.3", - "@nats-io/jetstream": "3.0.0", - "@nats-io/kv": "3.0.0", - "@nats-io/nats-core": "3.0.0", - "@nats-io/services": "3.0.0", "@orama/orama": "3.0.0-rc-3", "@react-hook/mouse-position": "^4.1.3", "@rinsuki/lz4-ts": "^1.0.1", - "@speed-highlight/core": "^1.1.11", "@stripe/react-stripe-js": "^3.1.1", "@stripe/stripe-js": "^5.5.0", "@types/debug": "^4.1.12", - "@uiw/react-textarea-code-editor": "^2.1.1", + "@uiw/react-textarea-code-editor": "^3.1.1", "@use-gesture/react": "^10.2.24", + "@xterm/addon-fit": "^0.10.0", + "@xterm/addon-web-links": "^0.11.0", + "@xterm/addon-webgl": "^0.18.0", + "@xterm/xterm": "^5.5.0", "@zippytech/react-notify-resize": "^4.0.4", "anser": "^2.1.1", "antd": "^5.24.7", @@ -78,16 +76,12 @@ "async": "^2.6.3", "audio-extensions": "^0.0.0", "awaiting": "^3.0.0", - "bootbox": "^4.4.0", - "bootstrap": "=3.4.1", - "bootstrap-colorpicker": "^2.5.3", "cat-names": "^3.1.0", "cheerio": "1.0.0-rc.10", "codemirror": "^5.65.18", "color-map": "^2.0.6", "copy-to-clipboard": "^3.0.8", "create-react-class": "^15.7.0", - "css-color-names": "0.0.4", "csv-parse": "^5.3.6", "csv-stringify": "^6.3.0", "d3": "^3.5.6", @@ -101,23 +95,13 @@ "entities": "^4.3.1", "escape-carriage": "^1.3.1", "events": "3.3.0", - "expect": "^26.6.2", - "fflate": "0.7.3", "gpt3-tokenizer": "^1.1.5", - "history": "^1.17.0", "html-react-parser": "^1.4.14", - "htmlparser": "^1.7.7", "humanize-list": "^1.0.1", "image-extensions": "^1.1.0", "immutable": "^4.3.0", - "install": "^0.13.0", "is-hotkey": "^0.2.0", "jquery": "^3.6.0", - "jquery-tooltip": "^0.2.1", - "jquery-ui": "^1.14.0", - "jquery-ui-touch-punch": "^0.2", - "jquery.payment": "^3.0.0", - "jquery.scrollintoview": "^1.9", "js-cookie": "^2.2.1", "json-stable-stringify": "^1.0.1", "jsonic": "^1.0.1", @@ -132,17 +116,12 @@ "md5": "^2", "memoize-one": "^5.1.1", "mermaid": "^11.4.1", - "nats.ws": "^1.30.2", "node-forge": "^1.0.0", - "octicons": "^3.5.0", "onecolor": "^3.1.0", "pdfjs-dist": "^4.6.82", - "pegjs": "^0.10.0", - "pica": "^7.1.0", "plotly.js": "^2.29.1", "project-name-generator": "^2.1.6", "prop-types": "^15.7.2", - "punycode": "2.3.1", "re-resizable": "^6.9.0", "react": "^18.3.1", "react-color": "^2.19.3", @@ -158,13 +137,9 @@ "react-redux": "^8.0.5", "react-timeago": "^7.2.0", "react-virtuoso": "^4.9.0", - "shallowequal": "^1.1.0", - "shell-escape": "^0.2.0", "slate": "^0.103.0", "superb": "^3.0.0", "three-ancient": "npm:three@=0.78.0", - "timeago": "^1.6.3", - "tslib": "^2.3.1", "underscore": "^1.12.1", "universal-cookie": "^4.0.4", "use-async-effect": "^2.2.7", @@ -174,30 +149,26 @@ "utility-types": "^3.10.0", "video-extensions": "^1.2.0", "xss": "^1.0.11", - "xterm": "5.0.0", - "xterm-addon-fit": "^0.6.0", - "xterm-addon-web-links": "^0.7.0", - "xterm-addon-webgl": "^0.13.0", "zlibjs": "^0.3.1" }, "devDependencies": { + "@cfaester/enzyme-adapter-react-18": "^0.8.0", "@cspell/dict-typescript": "^3.2.0", - "@formatjs/cli": "^6.2.12", + "@formatjs/cli": "^6.7.1", "@types/codemirror": "^5.60.15", "@types/jquery": "^3.5.5", "@types/katex": "^0.16.7", "@types/lodash": "^4.14.202", "@types/markdown-it": "12.2.3", "@types/md5": "^2.2.0", - "@types/mocha": "^10.0.0", - "@types/pica": "^5.1.3", "@types/react": "^18.3.10", "@types/react-dom": "^18.3.0", "@types/react-redux": "^7.1.25", "coffeescript": "^2.5.1", "cspell": "^8.17.2", - "mocha": "^10.0.0", + "enzyme": "^3.11.0", "react-test-renderer": "^18.2.0", + "tsd": "^0.22.0", "type-fest": "^3.3.0" }, "scripts": { @@ -205,7 +176,8 @@ "build": "rm -rf dist && pnpm i18n:compile && NODE_OPTIONS=--max-old-space-size=8192 tsc --build && coffee -m -c -o dist/ .", "build-coffee": "pnpm exec coffee -m -c -o dist/ .", "tsc": "NODE_OPTIONS=--max-old-space-size=8192 ../node_modules/.bin/tsc --watch --pretty --preserveWatchOutput ", - "test": "npx jest i18n/ misc/", + "test": "jest", + "depcheck": "pnpx depcheck --ignores cspell,@cspell/dict-typescript,events,@formatjs/cli | grep -Ev '\\.coffee|coffee$'", "prepublishOnly": "pnpm test", "update-color-scheme": "node ./update-color-scheme.js", "clean": "rm -rf dist node_modules i18n/trans/*.compiled.json", diff --git a/src/packages/frontend/project/context.tsx b/src/packages/frontend/project/context.tsx index 0b92e2d1ce..10d5a9e1bb 100644 --- a/src/packages/frontend/project/context.tsx +++ b/src/packages/frontend/project/context.tsx @@ -25,7 +25,6 @@ import { init as INIT_PROJECT_STATE, useProjectState, } from "./page/project-state-hook"; -import { useProjectStatus } from "./page/project-status-hook"; import { useProjectHasInternetAccess } from "./settings/has-internet-access-hook"; import { Project } from "./settings/types"; @@ -100,7 +99,6 @@ export function useProjectContextProvider({ const actions = useActions({ project_id }); const { project, group, compute_image } = useProject(project_id); const status: ProjectStatus = useProjectState(project_id); - useProjectStatus(actions); const hasInternet = useProjectHasInternetAccess(project_id); const isRunning = useMemo( () => status.get("state") === "running", diff --git a/src/packages/frontend/project/directory-listing.ts b/src/packages/frontend/project/directory-listing.ts index 08b08dc2f0..a3f0c00510 100644 --- a/src/packages/frontend/project/directory-listing.ts +++ b/src/packages/frontend/project/directory-listing.ts @@ -112,7 +112,7 @@ export async function get_directory_listing(opts: ListingOpts) { } } -import { Listings } from "@cocalc/frontend/nats/listings"; +import { Listings } from "@cocalc/frontend/conat/listings"; export async function get_directory_listing2(opts: ListingOpts): Promise { log("get_directory_listing2", opts); diff --git a/src/packages/frontend/project/explorer/file-listing/file-listing.tsx b/src/packages/frontend/project/explorer/file-listing/file-listing.tsx index 89f304a27e..4a75f3af42 100644 --- a/src/packages/frontend/project/explorer/file-listing/file-listing.tsx +++ b/src/packages/frontend/project/explorer/file-listing/file-listing.tsx @@ -23,7 +23,7 @@ import { useTypedRedux, } from "@cocalc/frontend/app-framework"; import useVirtuosoScrollHook from "@cocalc/frontend/components/virtuoso-scroll-hook"; -import { WATCH_THROTTLE_MS } from "@cocalc/frontend/nats/listings"; +import { WATCH_THROTTLE_MS } from "@cocalc/frontend/conat/listings"; import { ProjectActions } from "@cocalc/frontend/project_actions"; import { MainConfiguration } from "@cocalc/frontend/project_configuration"; import * as misc from "@cocalc/util/misc"; diff --git a/src/packages/frontend/project/explorer/misc-side-buttons.tsx b/src/packages/frontend/project/explorer/misc-side-buttons.tsx index c79e8772cb..fcc8a65e04 100644 --- a/src/packages/frontend/project/explorer/misc-side-buttons.tsx +++ b/src/packages/frontend/project/explorer/misc-side-buttons.tsx @@ -7,7 +7,6 @@ import { Space } from "antd"; import { join } from "path"; import React from "react"; import { defineMessage, useIntl } from "react-intl"; - import { Button, ButtonToolbar } from "@cocalc/frontend/antd-bootstrap"; import { Icon, Tip, VisibleLG } from "@cocalc/frontend/components"; import LinkRetry from "@cocalc/frontend/components/link-retry"; @@ -19,6 +18,8 @@ import { ProjectActions } from "@cocalc/frontend/project_store"; import track from "@cocalc/frontend/user-tracking"; import { KUCALC_COCALC_COM } from "@cocalc/util/db-schema/site-defaults"; +const SHOW_SERVER_LAUNCHERS = false; + import TourButton from "./tour/button"; const OPEN_MSG = defineMessage({ @@ -208,10 +209,12 @@ export const MiscSideButtons: React.FC = (props) => { style={{ whiteSpace: "nowrap", padding: "0" }} className="pull-right" > - - {render_jupyterlab_button()} - {render_vscode_button()} - + {SHOW_SERVER_LAUNCHERS && ( + + {render_jupyterlab_button()} + {render_vscode_button()} + + )} {render_upload_button()} {render_library_button()} diff --git a/src/packages/frontend/project/explorer/tour/button.tsx b/src/packages/frontend/project/explorer/tour/button.tsx index f07a3fe2d1..f3328e210e 100644 --- a/src/packages/frontend/project/explorer/tour/button.tsx +++ b/src/packages/frontend/project/explorer/tour/button.tsx @@ -11,7 +11,6 @@ export default function ProjectTourButton({ project_id }) { } return (
    ); diff --git a/src/packages/frontend/project/info/full.tsx b/src/packages/frontend/project/info/full.tsx index ae436f4d18..ed8ae52ade 100644 --- a/src/packages/frontend/project/info/full.tsx +++ b/src/packages/frontend/project/info/full.tsx @@ -7,12 +7,10 @@ declare let DEBUG; import { InfoCircleOutlined, ScheduleOutlined } from "@ant-design/icons"; import { Alert, Button, Form, Modal, Popconfirm, Switch, Table } from "antd"; - import { Col, Row } from "@cocalc/frontend/antd-bootstrap"; import { CSS, ProjectActions, redux } from "@cocalc/frontend/app-framework"; import { A, Loading, Tip } from "@cocalc/frontend/components"; import { SiteName } from "@cocalc/frontend/customize"; -import { ProjectInfo as WSProjectInfo } from "@cocalc/frontend/project/websocket/project-info"; import { Process, ProjectInfo as ProjectInfoType, @@ -20,7 +18,6 @@ import { import { field_cmp, seconds2hms } from "@cocalc/util/misc"; import { COLORS } from "@cocalc/util/theme"; import { RestartProject } from "../settings/restart-project"; -import { Channel } from "@cocalc/comm/websocket/types"; import { AboutContent, CGroup, @@ -36,13 +33,12 @@ import { ROOT_STYLE } from "../servers/consts"; interface Props { any_alerts: () => boolean; cg_info: CGroupInfo; - chan: Channel | null; render_disconnected: () => JSX.Element | undefined; disconnected: boolean; disk_usage: DUState; error: JSX.Element | null; status: string; - info: ProjectInfoType | undefined; + info: ProjectInfoType | null; loading: boolean; modal: string | Process | undefined; project_actions: ProjectActions | undefined; @@ -59,7 +55,6 @@ interface Props { show_explanation: boolean; show_long_loading: boolean; start_ts: number | undefined; - sync: WSProjectInfo | null; render_cocalc: (proc: ProcessRow) => JSX.Element | undefined; onCellProps; } @@ -68,7 +63,6 @@ export function Full(props: Readonly): JSX.Element { const { any_alerts, cg_info, - chan, render_disconnected, disconnected, disk_usage, @@ -91,7 +85,6 @@ export function Full(props: Readonly): JSX.Element { show_explanation, show_long_loading, start_ts, - sync, render_cocalc, onCellProps, } = props; @@ -287,7 +280,7 @@ export function Full(props: Readonly): JSX.Element { title={"The role of these processes in this project."} trigger={["hover", "click"]} > - + ); @@ -471,8 +464,7 @@ export function Full(props: Readonly): JSX.Element { ) : ( "no timestamp" )}{" "} - | Connections sync={`${sync != null}`} chan= - {`${chan != null}`} | Status: {status} + | Status: {status} ); } @@ -496,15 +488,6 @@ export function Full(props: Readonly): JSX.Element { ); } - function render_error() { - if (error == null) return; - return ( - - - - ); - } - function render_not_running() { if (project_state === "running") return; return ( @@ -521,7 +504,7 @@ export function Full(props: Readonly): JSX.Element { return (
    {render_not_running()} - {render_error()} + {error} {render_body()}
    ); diff --git a/src/packages/frontend/project/info/project-info.tsx b/src/packages/frontend/project/info/project-info.tsx index 1fb380b935..eab4241ea5 100644 --- a/src/packages/frontend/project/info/project-info.tsx +++ b/src/packages/frontend/project/info/project-info.tsx @@ -3,25 +3,17 @@ * License: MS-RSL – see LICENSE.md for details */ -declare let DEBUG; - import { Alert } from "antd"; -import { delay } from "awaiting"; - import { React, Rendered, redux, useActions, - useIsMountedRef, - useRef, + useMemo, useState, useTypedRedux, } from "@cocalc/frontend/app-framework"; import { useProjectContext } from "@cocalc/frontend/project/context"; -import { ProjectInfo as WSProjectInfo } from "@cocalc/frontend/project/websocket/project-info"; -import type { Channel } from "@cocalc/comm/websocket/types"; -import { webapp_client } from "@cocalc/frontend/webapp-client"; import { Process, ProjectInfo as ProjectInfoType, @@ -32,29 +24,9 @@ import { CoCalcFile, render_cocalc_btn } from "./components"; import { Flyout } from "./flyout"; import { Full } from "./full"; import { CGroupInfo, DUState, PTStats, ProcessRow } from "./types"; -import { - connect_ws, - grid_warning, - linearList, - process_tree, - sum_children, -} from "./utils"; - -// DEV: DEBUG is true, add some generic static values about CGroups, such that these elements show up in the UI -const DEV = DEBUG - ? { - cgroup: { - mem_stat: { - hierarchical_memory_limit: 1000, - total_rss: 550, - }, - cpu_usage: 12, // seconds - cpu_usage_rate: 0.8, // seconds / second - oom_kills: 1, - cpu_cores_limit: 1, - } as ProjectInfoType["cgroup"], - } - : undefined; +import { grid_warning, linearList, process_tree, sum_children } from "./utils"; +import useProjectInfo from "./use-project-info"; +import ShowError from "@cocalc/frontend/components/error"; interface Props { project_id: string; @@ -87,10 +59,18 @@ const pt_stats_init = { export const ProjectInfo: React.FC = React.memo( ({ mode = "full", wrap }: Props) => { - const isMountedRef = useIsMountedRef(); const { project_id } = useProjectContext(); + const { disconnected, info, error, setError } = useProjectInfo({ + project_id, + }); + const loading = info == null; + const status = disconnected ? "Connecting..." : "Connected"; const project_actions = useActions({ project_id }); - const [idle_timeout, set_idle_timeout] = useState(30 * 60); + + const idle_timeout = useMemo(() => { + return redux.getStore("projects").get_idle_timeout(project_id); + }, []); + const show_explanation = useTypedRedux({ project_id }, "show_project_info_explanation") ?? false; // this is @cocalc/conn/project-status/types::ProjectStatus @@ -99,24 +79,13 @@ export const ProjectInfo: React.FC = React.memo( const [project, set_project] = useState(project_map?.get(project_id)); const [project_state, set_project_state] = useState(); const [start_ts, set_start_ts] = useState(undefined); - const [info, set_info] = useState(undefined); const [ptree, set_ptree] = useState(undefined); const [pt_stats, set_pt_stats] = useState(pt_stats_init); - // chan: websocket channel to send commands to the project (for now) - const [chan, set_chan] = useState(null); - const chanRef = useRef(null); - // sync-object sending us the real-time data about the project - const [sync, set_sync] = useState(null); - const syncRef = useRef(null); - const [status, set_status] = useState("initializing…"); - const [loading, set_loading] = useState(true); - const [disconnected, set_disconnected] = useState(true); const [selected, set_selected] = useState([]); const [expanded, set_expanded] = useState([]); const [have_children, set_have_children] = useState([]); const [cg_info, set_cg_info] = useState(gc_info_init); const [disk_usage, set_disk_usage] = useState(du_init); - const [error, set_error] = useState(null); const [modal, set_modal] = useState( undefined, ); @@ -139,112 +108,12 @@ export const ProjectInfo: React.FC = React.memo( } }, [project]); - React.useEffect(() => { - chanRef.current = chan; - }, [chan]); - - React.useEffect(() => { - syncRef.current = sync; - }, [sync]); - - React.useEffect(() => { - set_disconnected(chan == null || sync == null); - }, [sync, chan]); - // used in render_not_loading_info() React.useEffect(() => { const timer = setTimeout(() => set_show_long_loading(true), 30000); return () => clearTimeout(timer); }, []); - async function connect() { - set_status("connecting…"); - try { - // the synctable for the project info - const info_sync = webapp_client.project_client.project_info(project_id); - - // this might fail if the project is not updated - const chan = await connect_ws(project_id); - if (!isMountedRef.current) return; - - const update = () => { - if (!isMountedRef.current) return; - const data = info_sync.get(); - if (data != null) { - set_info({ ...data.toJS(), ...DEV } as ProjectInfoType); - } - }; - - info_sync.once("change", function () { - if (!isMountedRef.current) return; - set_loading(false); - set_status("receiving…"); - }); - - info_sync.on("change", update); - info_sync.once("ready", update); - - chan.on("close", async function () { - if (!isMountedRef.current) return; - set_status("websocket closed: reconnecting in 3 seconds…"); - set_chan(null); - await delay(3000); - if (!isMountedRef.current) return; - set_status("websocket closed: reconnecting now…"); - const new_chan = await connect_ws(project_id); - if (!isMountedRef.current) { - // well, we got one but now we don't need it - new_chan.end(); - return; - } - set_status("websocket closed: got new connection…"); - set_chan(new_chan); - }); - - set_chan(chan); - set_sync(info_sync); - } catch (err) { - set_error( - <> - Project information setup problem: {`${err}`} - , - ); - return; - } - } - - // once when mounted - function get_idle_timeout() { - const ito = redux.getStore("projects").get_idle_timeout(project_id); - set_idle_timeout(ito); - } - - // each time the project state changes (including when mounted) we connect/reconnect - React.useEffect(() => { - if (project_state !== "running") return; - try { - connect(); - get_idle_timeout(); - return () => { - if (isMountedRef.current) { - set_status("closing connection"); - } - if (chanRef.current != null) { - if (chanRef.current.readyState === chanRef.current.OPEN) { - chanRef.current.end(); - } - } - if (syncRef.current != null) { - syncRef.current.close(); - } - }; - } catch (err) { - if (isMountedRef.current) { - set_status(`ERROR: ${err}`); - } - } - }, [project_state]); - function update_top(info: ProjectInfoType) { // this shouldn't be the case, but somehow I saw this happening once // the ProjectInfoType type is updated to refrect this edge case and here we bail out @@ -426,14 +295,17 @@ export const ProjectInfo: React.FC = React.memo( } } + const showError = ( + + ); + switch (mode) { case "flyout": return ( = React.memo( show_long_loading={show_long_loading} start_ts={start_ts} status={status} - sync={sync} render_disconnected={render_disconnected} render_cocalc={render_cocalc} onCellProps={onCellProps} @@ -463,10 +334,9 @@ export const ProjectInfo: React.FC = React.memo( = React.memo( show_long_loading={show_long_loading} start_ts={start_ts} status={status} - sync={sync} render_disconnected={render_disconnected} render_cocalc={render_cocalc} onCellProps={onCellProps} diff --git a/src/packages/frontend/project/info/use-project-info.ts b/src/packages/frontend/project/info/use-project-info.ts new file mode 100644 index 0000000000..662c95a51a --- /dev/null +++ b/src/packages/frontend/project/info/use-project-info.ts @@ -0,0 +1,51 @@ +/* +React hook that gives realtime information about a project. + +*/ + +import { useInterval } from "react-interval-hook"; +import { get, type ProjectInfo } from "@cocalc/conat/project/project-info"; +import { useEffect, useMemo, useState } from "react"; + +export default function useProjectInfo({ + project_id, + compute_server_id = 0, + interval = 4000, +}: { + project_id: string; + compute_server_id?: number; + interval?: number; +}): { + info: ProjectInfo | null; + error: string; + setError: (string) => void; + disconnected: boolean; +} { + const start = useMemo(() => Date.now(), []); + const [info, setInfo] = useState(null); + const [error, setError] = useState(""); + const [disconnected, setDisconnected] = useState(true); + const update = async () => { + // console.log("update", { project_id }); + try { + const info = await get({ project_id, compute_server_id }); + setInfo(info); + setDisconnected(false); + setError(""); + } catch (err) { + if (Date.now() - start >= interval * 2.1) { + console.log(`WARNING: project info -- ${err}`); + setError("Project info not available -- start the project"); + } + setDisconnected(true); + } + }; + + useInterval(update, interval); + + useEffect(() => { + update(); + }, [project_id, compute_server_id]); + + return { info, error, setError, disconnected }; +} diff --git a/src/packages/frontend/project/info/utils.ts b/src/packages/frontend/project/info/utils.ts index fb8f253714..1298f887fa 100644 --- a/src/packages/frontend/project/info/utils.ts +++ b/src/packages/frontend/project/info/utils.ts @@ -4,9 +4,7 @@ */ import { basename } from "path"; - import { CSS } from "@cocalc/frontend/app-framework"; -import { project_websocket } from "@cocalc/frontend/frame-editors/generic/client"; import { Process, Processes, State } from "@cocalc/util/types/project-info/types"; import { ALERT_DISK_FREE, @@ -73,12 +71,6 @@ export function grid_warning(val: number, max: number): CSS { return col != null ? col : {}; } -export async function connect_ws(project_id: string) { - const ws = await project_websocket(project_id); - const chan = await ws.api.project_info(); - return chan; -} - // filter for processes in process_tree function keep_proc(proc): boolean { if (proc.pid === 1) { diff --git a/src/packages/frontend/project/new/new-file-dropdown.tsx b/src/packages/frontend/project/new/new-file-dropdown.tsx index bed91aa682..a3b65eeb87 100644 --- a/src/packages/frontend/project/new/new-file-dropdown.tsx +++ b/src/packages/frontend/project/new/new-file-dropdown.tsx @@ -54,7 +54,7 @@ export function NewFileDropdown({ title ??= intl.formatMessage({ id: "project.new.new-file-dropdown.label", - defaultMessage: "File types...", + defaultMessage: "More File Types...", description: "Label on a button to create one of several additional file types", }); diff --git a/src/packages/frontend/project/new/new-file-page.tsx b/src/packages/frontend/project/new/new-file-page.tsx index c5bb296c3c..91ac3c695f 100644 --- a/src/packages/frontend/project/new/new-file-page.tsx +++ b/src/packages/frontend/project/new/new-file-page.tsx @@ -6,7 +6,6 @@ import { Button, Input, Modal, Space } from "antd"; import { useEffect, useRef, useState } from "react"; import { defineMessage, FormattedMessage, useIntl } from "react-intl"; - import { default_filename } from "@cocalc/frontend/account"; import { Alert, Col, Row } from "@cocalc/frontend/antd-bootstrap"; import { diff --git a/src/packages/frontend/project/open-file.ts b/src/packages/frontend/project/open-file.ts index 079f43df9d..8f46a3369e 100644 --- a/src/packages/frontend/project/open-file.ts +++ b/src/packages/frontend/project/open-file.ts @@ -449,7 +449,7 @@ async function convert_sagenb_worksheet( return filename.slice(0, filename.length - 3) + "sagews"; } -const log_open_time: { [path: string]: { id: string; start: Date } } = {}; +const log_open_time: { [path: string]: { id: string; start: number } } = {}; export function log_file_open( project_id: string, @@ -482,7 +482,7 @@ export function log_file_open( const key = `${project_id}-${path}`; log_open_time[key] = { id, - start: webapp_client.server_time(), + start: Date.now(), }; } } @@ -502,7 +502,7 @@ export function log_opened_time(project_id: string, path: string): void { // do not allow recording the time more than once, which would be weird. delete log_open_time[key]; const actions = redux.getProjectActions(project_id); - const time = webapp_client.server_time().valueOf() - start.valueOf(); + const time = Date.now() - start; actions.log({ time }, id); } diff --git a/src/packages/frontend/project/page/file-tab.tsx b/src/packages/frontend/project/page/file-tab.tsx index ced9f0bf87..e5c969a850 100644 --- a/src/packages/frontend/project/page/file-tab.tsx +++ b/src/packages/frontend/project/page/file-tab.tsx @@ -11,7 +11,6 @@ A single tab in a project. import { Popover, Tag } from "antd"; import { CSSProperties, ReactNode } from "react"; import { defineMessage, useIntl } from "react-intl"; - import { getAlertName } from "@cocalc/comm/project-status/types"; import { CSS, diff --git a/src/packages/frontend/project/page/file-tabs.tsx b/src/packages/frontend/project/page/file-tabs.tsx index 63f92789b0..0e1eaab893 100644 --- a/src/packages/frontend/project/page/file-tabs.tsx +++ b/src/packages/frontend/project/page/file-tabs.tsx @@ -9,7 +9,6 @@ Tabs for the open files in a project. import type { TabsProps } from "antd"; import { Tabs } from "antd"; - import { useActions } from "@cocalc/frontend/app-framework"; import { renderTabBar, @@ -21,6 +20,8 @@ import { EDITOR_PREFIX, path_to_tab } from "@cocalc/util/misc"; import { file_tab_labels } from "../file-tab-labels"; import { FileTab } from "./file-tab"; +const MIN_WIDTH = 48; + function Label({ path, project_id, label }) { const { width } = useItemContext(); const { active } = useSortable({ id: project_id }); @@ -31,7 +32,11 @@ function Label({ path, project_id, label }) { path={path} label={label} noPopover={active != null} - style={width != null ? { width, marginRight: "-10px" } : undefined} + style={{ + ...(width != null + ? { width: Math.max(MIN_WIDTH, width + 15), marginRight: "-10px" } + : undefined), + }} /> ); } @@ -143,7 +148,11 @@ export default function FileTabs({ openFiles, project_id, activeTab }) { (false); const [error, setError] = useState(""); const syncRef = useRef(sync); const compute_server_id = useTypedRedux({ project_id }, "compute_server_id"); @@ -103,6 +104,7 @@ export function TerminalFlyout({ terminalRef.current.conn_write({ cmd: "size", rows: 0, cols: 0 }); terminalRef.current.close(); terminalRef.current = undefined; + setTerminalExists(false); } function getMockTerminalActions(): ConnectedTerminalInterface { @@ -191,6 +193,7 @@ export function TerminalFlyout({ } try { terminalRef.current = getTerminal(id, node); + setTerminalExists(true); } catch (err) { return; // not yet ready -- might be ok } @@ -222,14 +225,17 @@ export function TerminalFlyout({ // or switches to being visible and was not initialized. // See https://github.com/sagemathinc/cocalc/issues/5133 if (terminalRef.current != null || !is_visible) return; - init_terminal(); + // wait until is actually in the DOM before trying to render, + // or it will crash for sure (due to changes in @xterm) + setTimeout(init_terminal, 0); }, [is_visible]); useEffect(() => { // defensive, like with the frame terminal -- see https://github.com/sagemathinc/cocalc/issues/3819 if (terminalRef.current == null) return; delete_terminal(); - init_terminal(); + // see comment about regarding the setTimeout + setTimeout(init_terminal, 0); }, [id]); // resize is a counter, increases with debouncing, if size change. @@ -243,7 +249,7 @@ export function TerminalFlyout({ // the terminal follows changing the directory useEffect(() => { - if (terminalRef.current == null) return; + if (terminalRef.current == null || !terminalExists) return; if (syncPath === prevSyncPath && !sync) return; // this "line reset" is from the terminal guide, // see frame-editors/terminal-editor/actions::run_command @@ -253,7 +259,7 @@ export function TerminalFlyout({ const cmd = ` cd "$HOME/${nextCwd}"`; // this will end up in a write buffer, hence it should be ok to do right at the beginning terminalRef.current.conn_write(`${clean}${cmd}\n`); - }, [current_path, syncPath, sync]); + }, [current_path, syncPath, sync, terminalExists]); const set_font_size = debounce( () => { diff --git a/src/packages/frontend/project/page/flyouts/files.tsx b/src/packages/frontend/project/page/flyouts/files.tsx index 5325714c40..d186f049e7 100644 --- a/src/packages/frontend/project/page/flyouts/files.tsx +++ b/src/packages/frontend/project/page/flyouts/files.tsx @@ -28,7 +28,6 @@ import { useStudentProjectFunctionality } from "@cocalc/frontend/course"; import { file_options } from "@cocalc/frontend/editor-tmp"; import { FileUploadWrapper } from "@cocalc/frontend/file-upload"; import { should_open_in_foreground } from "@cocalc/frontend/lib/should-open-in-foreground"; -import { WATCH_THROTTLE_MS } from "@cocalc/frontend/nats/listings"; import { useProjectContext } from "@cocalc/frontend/project/context"; import { compute_file_masks } from "@cocalc/frontend/project/explorer/compute-file-masks"; import { @@ -36,6 +35,7 @@ import { DirectoryListingEntry, FileMap, } from "@cocalc/frontend/project/explorer/types"; +import { WATCH_THROTTLE_MS } from "@cocalc/frontend/conat/listings"; import { mutate_data_to_compute_public_files } from "@cocalc/frontend/project_store"; import track from "@cocalc/frontend/user-tracking"; import { diff --git a/src/packages/frontend/project/page/project-status-hook.ts b/src/packages/frontend/project/page/project-status-hook.ts deleted file mode 100644 index be13c434cd..0000000000 --- a/src/packages/frontend/project/page/project-status-hook.ts +++ /dev/null @@ -1,55 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ -import { React } from "@cocalc/frontend/app-framework"; -import { webapp_client } from "@cocalc/frontend/webapp-client"; -import { ProjectStatus as WSProjectStatus } from "../websocket/project-status"; -import { ProjectStatus } from "@cocalc/comm/project-status/types"; -import { ProjectActions } from "@cocalc/frontend/project_actions"; -import { useProjectState } from "./project-state-hook"; - -// this records data from the synctable "project_status" in redux. -// used in page/page when a project is added to the UI -// if you want to know the project state, do -// const project_status = useTypedRedux({ project_id }, "status"); -export function useProjectStatus(actions?: ProjectActions): void { - const project_id: string | undefined = actions?.project_id; - const statusRef = React.useRef(null); - const project_state = useProjectState(project_id); - - function set_status(status) { - actions?.setState({ status }); - } - - function connect() { - if (project_id == null) return; - const status_sync = webapp_client.project_client.project_status(project_id); - statusRef.current = status_sync; - const update = () => { - const data = status_sync.get(); - if (data != null) { - set_status(data.toJS() as ProjectStatus); - } else { - // For debugging: - // console.warn(`status_sync ${project_id}: got no data`); - } - }; - status_sync.once("ready", update); - status_sync.on("change", update); - } - - // each time the project state changes to running (including when mounted) we connect/reconnect - React.useEffect(() => { - if (project_state == null) return; - if (project_state.get("state") !== "running") return; - try { - connect(); - return () => { - statusRef.current?.close(); - }; - } catch (err) { - console.warn(`status_sync ${project_id} error: ${err}`); - } - }, [project_state]); -} diff --git a/src/packages/frontend/project/websocket/api.ts b/src/packages/frontend/project/websocket/api.ts index 2597e81e56..00ed94d1a6 100644 --- a/src/packages/frontend/project/websocket/api.ts +++ b/src/packages/frontend/project/websocket/api.ts @@ -29,30 +29,26 @@ import type { Mesg, NbconvertParams, } from "@cocalc/comm/websocket/types"; -import call from "@cocalc/sync/client/call"; import { webapp_client } from "@cocalc/frontend/webapp-client"; -import { type ProjectApi } from "@cocalc/nats/project-api"; +import { type ProjectApi } from "@cocalc/conat/project/api"; import type { ExecuteCodeOutput, ExecuteCodeOptions, } from "@cocalc/util/types/execute-code"; -import { formatterClient } from "@cocalc/nats/service/formatter"; -import { syncFsClientClient } from "@cocalc/nats/service/syncfs-client"; -// import { syncFsServerClient } from "@cocalc/nats/service/syncfs-server"; +import { formatterClient } from "@cocalc/conat/service/formatter"; +import { syncFsClientClient } from "@cocalc/conat/service/syncfs-client"; + +const log = (...args) => { + console.log("project:websocket: ", ...args); +}; export class API { - private conn; private project_id: string; - private cachedVersion?: number; private apiCache: { [key: string]: ProjectApi } = {}; - constructor(conn, project_id: string) { - this.conn = conn; + constructor(project_id: string) { this.project_id = project_id; this.listing = reuseInFlight(this.listing.bind(this)); - this.conn.on("end", () => { - delete this.cachedVersion; - }); } private getApi = ({ @@ -68,7 +64,7 @@ export class API { } const key = `${compute_server_id}-${timeout}`; if (this.apiCache[key] == null) { - this.apiCache[key] = webapp_client.nats_client.projectApi({ + this.apiCache[key] = webapp_client.conat_client.projectApi({ project_id: this.project_id, compute_server_id, timeout, @@ -77,38 +73,35 @@ export class API { return this.apiCache[key]!; }; - private primusCall = async (mesg: Mesg, timeout: number) => { - return await call(this.conn, mesg, timeout); - }; - - private _call = async (mesg: Mesg, timeout: number): Promise => { - return await webapp_client.nats_client.projectWebsocketApi({ + private _call = async ( + mesg: Mesg, + timeout: number, + compute_server_id = 0, + ): Promise => { + log("_call (NEW conat call)", mesg); + const resp = await webapp_client.conat_client.projectWebsocketApi({ project_id: this.project_id, + compute_server_id, mesg, timeout, }); + log("_call worked and returned", resp); + return resp; }; - private getChannel = async (channel_name: string) => { - const natsConn = await webapp_client.nats_client.primus(this.project_id); - // TODO -- typing - return natsConn.channel(channel_name) as unknown as Channel; + private getChannel = ( + channel: string, + compute_server_id?: number, + ): Channel => { + return webapp_client.conat_client.primus({ + project_id: this.project_id, + compute_server_id, + channel, + }) as unknown as Channel; }; call = async (mesg: Mesg, timeout: number) => { - try { - return await this._call(mesg, timeout); - } catch (err) { - if (err.code == "PERMISSIONS_VIOLATION") { - // request update of our credentials to include this project, then try again - await webapp_client.nats_client.addProjectPermissions([ - this.project_id, - ]); - return await this._call(mesg, timeout); - } else { - throw err; - } - } + return await this._call(mesg, timeout); }; getComputeServerId = (path: string) => { @@ -368,39 +361,9 @@ export class API { return await api.editor.jupyterRunNotebook(opts); }; - // TODO! - terminal = async (path: string, options: object = {}): Promise => { - const channel_name = await this.call( - { - cmd: "terminal", - path, - options, - }, - 20000, - ); - return await this.getChannel(channel_name); - }; - - project_info = async (): Promise => { - const channel_name = await this.primusCall({ cmd: "project_info" }, 60000); - return await this.getChannel(channel_name); - }; - - // Get the lean *channel* for the given '.lean' path. - lean_channel = async (path: string): Promise => { - const channel_name = await this.primusCall( - { - cmd: "lean_channel", - path: path, - }, - 60000, - ); - return this.conn.channel(channel_name); - }; - // Get the x11 *channel* for the given '.x11' path. x11_channel = async (path: string, display: number): Promise => { - const channel_name = await this.primusCall( + const channel_name = await this._call( { cmd: "x11_channel", path, @@ -408,57 +371,8 @@ export class API { }, 60000, ); - return this.conn.channel(channel_name); - }; - - // Get the sync *channel* for the given SyncTable project query. - synctable_channel = async ( - query: { [field: string]: any }, - options: { [field: string]: any }[], - ): Promise => { - const channel_name = await this.primusCall( - { - cmd: "synctable_channel", - query, - options, - }, - 10000, - ); - // console.log("synctable_channel", query, options, channel_name); - return this.conn.channel(channel_name); - }; - - // Command-response API for synctables. - // - mesg = {cmd:'close'} -- closes the synctable, even if persistent. - syncdoc_call = async ( - path: string, - mesg: { [field: string]: any }, - timeout_ms: number = 30000, // ms timeout for call - ): Promise => { - return await this.call({ cmd: "syncdoc_call", path, mesg }, timeout_ms); - }; - - // Do a request/response command to the lean server. - lean = async (opts: any): Promise => { - let timeout_ms = 10000; - if (opts.timeout) { - timeout_ms = opts.timeout * 1000 + 2000; - } - return await this.call({ cmd: "lean", opts }, timeout_ms); - }; - - // I think this isn't used. It was going to support - // sync_channel, but obviously a more nuanced protocol - // was required. - symmetric_channel = async (name: string): Promise => { - const channel_name = await this.primusCall( - { - cmd: "symmetric_channel", - name, - }, - 30000, - ); - return this.conn.channel(channel_name); + log("x11_channel"); + return this.getChannel(channel_name); }; // Copying files to/from compute servers: diff --git a/src/packages/frontend/project/websocket/connect.ts b/src/packages/frontend/project/websocket/connect.ts index 7b558061b8..1a19af18b6 100644 --- a/src/packages/frontend/project/websocket/connect.ts +++ b/src/packages/frontend/project/websocket/connect.ts @@ -12,257 +12,33 @@ wat once, and hence we make many Primus websocket connections simultaneously to the same domain. It does work, but not without an ugly hack. */ -import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; -import { callback, delay } from "awaiting"; -import { ajax, globalEval } from "jquery"; -import { join } from "path"; -import { redux } from "@cocalc/frontend/app-framework"; -import { appBasePath } from "@cocalc/frontend/customize/app-base-path"; -import { webapp_client } from "@cocalc/frontend/webapp-client"; -import { allow_project_to_run } from "../client-side-throttle"; import { API } from "./api"; -import { set_project_websocket_state, WebsocketState } from "./websocket-state"; -const { - callback2, - once, - retry_until_success, -} = require("@cocalc/util/async-utils"); // so also works on backend. +import { set_project_websocket_state } from "./websocket-state"; const connections = {}; -// This is a horrible temporary hack to ensure that we do not load two global Primus -// client libraries at the same time, with one overwriting the other with the URL -// of the target, hence causing multiple projects to have the same websocket. -let READING_PRIMUS_JS = false; - -async function wait_for_project_to_start(project_id: string) { - if (!(await allow_project_to_run(project_id))) { - throw Error("not allowing right now"); - } - // also check if the project is supposedly running and if - // not wait for it to be. - const projects = redux.getStore("projects"); - if (projects == null) { - throw Error("projects store must exist"); - } - - if (!projects.is_collaborator(project_id)) { - // wait below not useful: - return; +import { EventEmitter } from "events"; +class FakeConn extends EventEmitter { + public api: API; + constructor(project_id) { + super(); + this.api = new API(project_id); + set_project_websocket_state(project_id, "online"); } - await callback2(projects.wait, { - until: () => projects.get_state(project_id) == "running", - }); + destroy = () => {}; } -async function connection_to_project0(project_id: string): Promise { +export async function connection_to_project(project_id: string): Promise { if (project_id == null || project_id.length != 36) { throw Error(`project_id (="${project_id}") must be a valid uuid`); } if (connections[project_id] !== undefined) { return connections[project_id]; } - - function log(..._args): void { - // Uncomment for very verbose logging/debugging... - console.log(`project websocket("${project_id}")`, ..._args); - } - log("connecting..."); - const window0: any = (global as any).window as any; // global part is so this also compiles on node.js. - const url: string = join(appBasePath, project_id, "raw/.smc/primus.js"); - - const Primus0 = window0.Primus; // the global primus - let Primus; - - // So that the store reflects that we are not connected but are trying. - set_project_websocket_state(project_id, "offline"); - - const MAX_AJAX_TIMEOUT_MS: number = 10000; - - async function get_primus(do_eval: boolean) { - let timeout: number = 7500; - await retry_until_success({ - f: async function () { - if (do_eval && READING_PRIMUS_JS) { - throw Error("currently reading one already"); - } - - if (!webapp_client.is_signed_in()) { - // At least wait until main client is signed in, since nothing - // will work until that is the case anyways. - await once(webapp_client, "signed_in"); - } - - log("wait_for_project_to_start..."); - await wait_for_project_to_start(project_id); - log("wait_for_project_to_start: done"); - - // Now project is thought to be running, so maybe this will work: - try { - if (do_eval) { - READING_PRIMUS_JS = true; - } - - /* - We use a timeout in the ajax call before, since while the project is - starting up the call ends up taking a LONG time to "Stall out" due to settings - in a proxy server somewhere along the way. This makes the project start time - (i.e., how long until websocket is working) seem really slow for no good reason. - Instead, we keep retrying the primus.js GET request pretty aggressively until - success. - NOTE: there is the real potential of very slow 3G clients not being able to complete the - GET, which is why we increase it each time up to MAX_AJAX_TIMEOUT_MS. - */ - - const load_primus = (cb) => { - ajax({ - timeout, - type: "GET", - url, - // text, in contrast to "script", doesn't eval it -- we do that! - dataType: "text", - error: () => { - cb("ajax error -- try again"); - }, - success: async function (data) { - // console.log("success. data:", data.slice(0, 100)); - if (data.charAt(0) !== "<") { - if (do_eval) { - try { - await globalEval(data); - } catch (err) { - cb(err); - return; - } - } - cb(); - } else { - cb("wrong data -- try again"); - } - }, - }); - }; - log( - `load_primus: attempt to get primus.js with timeout=${timeout}ms and do_eval=${do_eval}`, - ); - await callback(load_primus); - log("load_primus: done"); - - if (do_eval) { - Primus = window0.Primus; - window0.Primus = Primus0; // restore global primus - } - } finally { - if (do_eval) { - READING_PRIMUS_JS = false; - } - timeout = Math.min(timeout * 1.2, MAX_AJAX_TIMEOUT_MS); - //console.log("success!"); - } - }, - start_delay: 1000, - max_delay: 10000, // do not make too aggressive or it DDOS proxy server; - // but also not too slow since project startup will feel slow to user. - // NOTE that since we wait until the project is running before any attempt to connect, - // most of the time we do a GET request, then wait for it to fail, which takes timeout ms. - // This delay here (that retry_until_success introduces) is really only due to - // paranoia that maybe the GET request fails very quickly (I don't know if that - // is even possible). - factor: 1.3, - desc: "connecting to project", - // log: (...x) => { - // log("retry primus:", ...x); - // }, - }); - - log("got primus.js successfully"); - } - await get_primus(true); - - // This dance is because evaling primus_js sets window.Primus. - // However, we don't want to overwrite the usual global window.Primus. - // Also, we use {strategy:false} to **completely disable** all - // automatic reconnect logic (see https://github.com/primus/primus#strategy), - // because of recent bugs (optimizations?) in web browsers that make it - // so after a certain number of failed reconnect attempts, they totally BREAK - // and you have to restart your browser complete (not good). - const conn = (connections[project_id] = Primus.connect({ - strategy: false, - manual: true, - })); - conn.open(); - - conn.api = new API(conn, project_id); - conn.verbose = false; - - // Given conn a state API, which is very handy for my use. - // This both emits something (useful for sync and other code), - // and sets information in the projects store (useful for UI). - - // And also some logging to the console about what is - // going on in some cases. - - function update_state(state: WebsocketState): void { - if (conn.state == state) { - return; // nothing changed, so no need to set or emit. - } - //console.log( - // `project websocket: state='${state}', project_id='${project_id}'` - //); - conn.state = state; - conn.emit("state", state); - set_project_websocket_state(project_id, state); - } - update_state("offline"); // starts offline - - conn.on("open", () => { - log("online!"); - update_state("online"); - }); - - /* - CRITICAL: do NOT consider this as online -- conn emits - online before open, and this causes havoc with synctable. - conn.on("online", () => { - update_state("online"); - });*/ - - conn.on("offline", () => { - update_state("offline"); - }); - - conn.on("destroy", () => { - update_state("destroyed"); - }); - - // Instead of using the primus reconnect logic, which just keeps - // attempting websocket connections (which turns out to be very bad - // for modern browsers!), we use our own strategy. - conn.on("end", async function () { - while (webapp_client.idle_client.inStandby()) { - await delay(1000); - } - log(`project websocket: reconnecting to '${project_id}' in 1s...`); - // put this delay in since otherwise we try to reconnect so rapidly that - // we basically DOS the project thus slowing down connecting by a - // few seconds, which is dumb: - await delay(1000); - if (conn.api == null) return; // done with this connection - update_state("offline"); - await get_primus(false); - if (conn.api == null) return; // done with this connection - conn.open(); - }); - - // conn.on("data", (data) => { - // console.log("project websocket received data", data); - // }); - - return conn; + connections[project_id] = new FakeConn(project_id); + return connections[project_id]; } -export const connection_to_project = reuseInFlight(connection_to_project0); - export function disconnect_from_project(project_id: string): void { const conn = connections[project_id]; if (conn === undefined) { diff --git a/src/packages/frontend/project/websocket/listings.ts b/src/packages/frontend/project/websocket/listings.ts deleted file mode 100644 index 5908e08497..0000000000 --- a/src/packages/frontend/project/websocket/listings.ts +++ /dev/null @@ -1,510 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -import { redux, TypedMap } from "@cocalc/frontend/app-framework"; -import { exec, query } from "@cocalc/frontend/frame-editors/generic/client"; -import { webapp_client } from "@cocalc/frontend/webapp-client"; -import { SyncTable } from "@cocalc/sync/table"; -import { once } from "@cocalc/util/async-utils"; -import { WATCH_TIMEOUT_MS, Listing } from "@cocalc/util/db-schema/listings"; -import { deleted_file_variations } from "@cocalc/util/delete-files"; -import type { DirectoryListingEntry } from "@cocalc/util/types"; -import { close, path_split } from "@cocalc/util/misc"; -import { delay } from "awaiting"; -import { EventEmitter } from "events"; -import { fromJS, List } from "immutable"; -import { throttle } from "lodash"; -import { get_directory_listing } from "../directory-listing"; -export const WATCH_THROTTLE_MS = WATCH_TIMEOUT_MS / 2; - -type ImmutablePathEntry = TypedMap; - -type State = "init" | "ready" | "closed"; - -export type ImmutableListing = TypedMap; - -export class Listings extends EventEmitter { - private table?: SyncTable; - private project_id: string; - private compute_server_id: number; - private last_version: { [path: string]: any } = {}; // last version emitted via change event. - private last_deleted: { [path: string]: any } = {}; - private state: State = "init"; - private throttled_watch: { [path: string]: Function } = {}; - - constructor(project_id: string, compute_server_id: number = 0) { - super(); - this.project_id = project_id; - this.compute_server_id = compute_server_id; - this.init(); - } - - // Watch directory for changes. - // IMPORTANT: This may and must be called frequently, e.g., at least - // once every 45 seconds. The point is to convey to the backend - // that at least one client is interested in this path. - // Don't worry about calling this function **too much**, since - // it throttles calls. - watch = async (path: string, force: boolean = false): Promise => { - if (force) { - await this._watch(path); - return; - } - if (this.throttled_watch[path] == null) { - this.throttled_watch[path] = throttle( - () => this._watch(path), - WATCH_THROTTLE_MS, - { - leading: true, - trailing: true, - }, - ); - } - if (this.throttled_watch[path] == null) throw Error("bug"); - this.throttled_watch[path](); - }; - - private _watch = async (path: string): Promise => { - if (await this.waitUntilReady(false)) return; - if (this.state == "closed") return; - this.set({ - path, - interest: webapp_client.server_time(), - }); - }; - - get = async ( - path: string, - trigger_start_project?: boolean, - ): Promise => { - if (this.state != "ready") { - try { - const listing = await this.getUsingDatabase(path); - if (listing != null) { - return listing; - } - } catch (err) { - // ignore -- e.g., maybe user doesn't have access or db not available. Fine either way. - } - } - if (this.state != "ready") { - // State still not ready and nothing in the database. - // If project is running, try directly getting listing (this is meant - // for old projects that haven't been restarted since we released the new - // sync code, but could possibly be a useful fallback in case of other - // problems). - const listing = await this.getListingDirectly( - path, - trigger_start_project, - ); - if (listing != null) { - return listing; - } - } - - const x = this.getRecord(path); - if (x == null || x.get("error")) return; - return x.get("listing")?.toJS() as any; - }; - - getDeleted = async (path: string): Promise | undefined> => { - if (this.state == "closed") return; - if (this.state != "ready") { - const q = await query({ - query: { - listings: { - project_id: this.project_id, - compute_server_id: this.compute_server_id, - path, - deleted: null, - }, - }, - }); - if (q.query.listings?.error) { - throw Error(q.query.listings?.error); - } - if (q.query.listings?.deleted != null) { - return fromJS(q.query.listings.deleted); - } else { - return; - } - } - if (this.state == ("closed" as State)) return; - if (this.state != ("ready" as State)) { - await once(this, "state"); - if (this.state != ("ready" as State)) return; - } - return this.getRecord(path)?.get("deleted"); - }; - - undelete = async (path: string): Promise => { - if (path == "") return; - if (this.state == ("closed" as State)) return; - if (this.state != ("ready" as State)) { - await once(this, "state"); - if (this.state != ("ready" as State)) return; - } - // Check isDeleted, so we can assume that path definitely - // is deleted according to our rules. - if (!this.isDeleted(path)) { - return; - } - - const { head, tail } = path_split(path); - if (head != "") { - // make sure the containing directory exists. - await exec({ - project_id: this.project_id, - command: "mkdir", - args: ["-p", head], - }); - } - const cur = this.getRecord(head); - if (cur == null) { - // undeleting a file that was maybe deleted as part of a directory tree. - // NOTE: If you undelete *one* file from directory tree, then - // creating any other file in that tree will just work. This is - // **by design** to keep things from getting too complicated! - await this.undelete(head); - return; - } - let deleted = cur.get("deleted"); - if (deleted == null || deleted.indexOf(tail) == -1) { - await this.undelete(head); - return; - } - const remove = new Set([tail].concat(deleted_file_variations(tail))); - const newDeleted = deleted.toJS().filter((x) => !remove.has(x)); - await this.set({ path: head, deleted: newDeleted }); - }; - - // true or false if known deleted or not; undefined if don't know yet. - // TODO: technically we should check the all the - // deleted_file_variations... but that is really an edge case - // that probably doesn't matter much. - public isDeleted = (filename: string): boolean | undefined => { - const { head, tail } = path_split(filename); - if (head != "" && this.isDeleted(head)) { - // recursively check if filename is contained in a - // directory tree that got deleted. - return true; - } - let x; - try { - x = this.getRecord(head); - } catch (err) { - return undefined; - } - if (x == null) return false; - const deleted = x.get("deleted"); - if (deleted == null) return false; - return deleted.indexOf(tail) != -1; - }; - - // Does a call to the project to directly determine whether or - // not the given path exists. This doesn't depend on the table. - // Can throw an exception if it can't contact the project. - exists = async (path: string): Promise => { - return ( - ( - await webapp_client.exec({ - project_id: this.project_id, - command: "test", - args: ["-e", path], - err_on_exit: false, - }) - ).exit_code == 0 - ); - }; - - // Returns: - // - List in case of a proper directory listing - // - string in case of an error - // - undefined if directory listing not known (and error not known either). - getForStore = async ( - path: string, - ): Promise | undefined | string> => { - if (this.state != "ready") { - const x = await this.getUsingDatabase(path); - if (x == null) return x; - return fromJS(x) as any; - } - const x = this.getRecord(path); - if (x == null) return x; - if (x.get("error")) { - return x.get("error"); - } - return x.get("listing"); - }; - - getUsingDatabase = async ( - path: string, - ): Promise => { - const q = await query({ - query: { - listings: { - project_id: this.project_id, - compute_server_id: this.compute_server_id, - path, - listing: null, - error: null, - }, - }, - }); - if (q.query.listings?.error) { - throw Error(q.query.listings?.error); - } - return q.query.listings?.listing; - }; - - getMissingUsingDatabase = async ( - path: string, - ): Promise => { - const q = await query({ - query: { - listings: { - project_id: this.project_id, - compute_server_id: this.compute_server_id, - path, - missing: null, - }, - }, - }); - return q.query.listings?.missing; - }; - - getMissing = (path: string): number | undefined => { - if (this.state != "ready") { - return; - } - const missing = this.getTable() - .get(JSON.stringify([this.project_id, path, this.compute_server_id])) - ?.get("missing"); - return missing; - }; - - getListingDirectly = async ( - path: string, - trigger_start_project?: boolean, - ): Promise => { - const store = redux.getStore("projects"); - // make sure that our relationship to this project is known. - if (store == null) throw Error("bug"); - const group = await store.async_wait({ - until: (s) => (s as any).get_my_group(this.project_id), - timeout: 60, - }); - const x = await get_directory_listing({ - project_id: this.project_id, - path, - hidden: true, - max_time_s: 15 * 60, - group, - trigger_start_project, - compute_server_id: this.compute_server_id, - }); - if (x.error != null) { - throw Error(x.error); - } else { - return x.files; - } - }; - - close = (): void => { - this.setState("closed"); - if (this.table != null) { - this.table.close(); - } - this.removeAllListeners(); - close(this); - this.setState("closed"); - }; - - // This is used to possibly work around a rare bug. - // https://github.com/sagemathinc/cocalc/issues/4790 - private reInit = async (): Promise => { - this.state = "init"; - await this.init(); - }; - - private init = async (): Promise => { - if (this.state != "init") { - throw Error("must be in init state"); - } - // Make sure there is a working websocket to the project - while (true) { - try { - await webapp_client.project_client.websocket(this.project_id); - break; - } catch (_) { - if (this.state == ("closed" as State)) return; - await delay(3000); - } - } - if ((this.state as State) == "closed") return; - - // Now create the table. - this.table = await webapp_client.sync_client.synctable_project( - this.project_id, - { - listings: [ - { - project_id: this.project_id, - compute_server_id: this.compute_server_id, - path: null, - listing: null, - time: null, - interest: null, - missing: null, - error: null, - deleted: null, - }, - ], - }, - [], - // space out individual changes by 100ms -- without this we get multiple 'change' events - // at the exact same time, which breaks properly updating the directory_listings data - // structure by the project store. This also helps avoid too much updating at once - // of the frontend UI. Symptom without this: refresh browser at HOME, then change to - // a subdir such as tmp and see it spin forever. - 100, - ); - - if ((this.state as State) == "closed") return; - - this.table.on("change", async (keys: string[]) => { - if (this.state != "ready") { - // don't do anything if being initialized or already closed, - // since code below will break in weird ways. - return; - } - // handle changes to directory listings and deleted files lists - const paths: string[] = []; - const deleted_paths: string[] = []; - for (const key of keys) { - const path = JSON.parse(key)[1]; - // Be careful to only emit a change event if the actual - // listing itself changes. Table emits more frequently, - // e.g., due to updating watch, time of listing changing, etc. - const this_version = await this.getForStore(path); - if (this_version != this.last_version[path]) { - this.last_version[path] = this_version; - paths.push(path); - } - - const this_deleted = this.getRecord(path)?.get("deleted"); - if (this_deleted != this.last_deleted[path]) { - if ( - this_deleted != null && - !this_deleted.equals(this.last_deleted[path]) - ) { - deleted_paths.push(path); - } - - this.last_deleted[path] = this_deleted; - } - } - if (paths.length > 0) { - this.emit("change", paths); - } - - if (deleted_paths.length > 0) { - this.emit("deleted", deleted_paths); - } - }); - this.setState("ready"); - }; - - private getTable = (): SyncTable => { - if (this.state != "ready") { - throw Error("table not initialized "); - } - if (this.table == null) { - throw Error("table is null"); - } - if (this.table.get_state() == "closed") { - throw Error("table is closed"); - } - return this.table; - }; - - private set = async (obj: Listing): Promise => { - let table; - try { - table = this.getTable(); - } catch (err) { - // See https://github.com/sagemathinc/cocalc/issues/4790 - console.warn("Error getting table -- ", err); - await this.reInit(); - table = this.getTable(); - } - const x = { - project_id: this.project_id, - compute_server_id: this.compute_server_id, - ...obj, - }; - // do NOT do the default deep merge, - // since things like the deleted list - // merge in a weird way. - table.set(x, "shallow"); - await table.save(); - }; - - isReady = (): boolean => { - return this.state == ("ready" as State); - }; - - private getRecord = (path: string): ImmutableListing | undefined => { - const x = this.getTable().get( - JSON.stringify([this.project_id, path, this.compute_server_id]), - ); - if (x == null) return x; - return x as unknown as ImmutableListing; // coercing to fight typescript. - // NOTE: That we have to use JSON.stringify above is an ugly shortcoming - // of the get method in @cocalc/sync/table/synctable.ts - // that could probably be relatively easily fixed. - }; - - private setState = (state: State): void => { - if (this.state == state) return; - if (this.state == "closed") { - throw Error("cannot switch away from closed"); - } - if (this.state == "ready" && state != "closed") { - throw Error("can only transition from ready to closed"); - } - this.state = state; - this.emit("state", state); - }; - - // Returns true if never will be ready - private waitUntilReady = async ( - exception: boolean = true, - ): Promise => { - try { - if (this.state == "closed") { - throw Error("Listings object must not be closed"); - } - if (this.state == "init") { - await once(this, "state"); - if ((this.state as State) != "ready") { - throw Error("never will be ready"); - } - return false; - } - return false; - } catch (err) { - if (exception) throw err; - return true; - } - }; -} - -export function listings( - project_id: string, - compute_server_id: number = 0, -): Listings { - return new Listings(project_id, compute_server_id); -} diff --git a/src/packages/frontend/project/websocket/project-info.ts b/src/packages/frontend/project/websocket/project-info.ts deleted file mode 100644 index 8fd0499ef0..0000000000 --- a/src/packages/frontend/project/websocket/project-info.ts +++ /dev/null @@ -1,155 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -/* -SyncTable of project information about a project. - -Use this via - - webapp_client.project_client.project_info(project_id) - -*/ - -import { EventEmitter } from "events"; -import { fromJS, Map } from "immutable"; -import { SyncTable } from "@cocalc/sync/table"; -import { delay } from "awaiting"; -import { close } from "@cocalc/util/misc"; -import { WebappClient } from "../../webapp-client"; - -type State = "init" | "ready" | "closed"; -type Info = Map; - -export class ProjectInfo extends EventEmitter { - private table?: SyncTable; - private project_id: string; - private state: State = "init"; - private client: WebappClient; - - constructor(client: WebappClient, project_id: string) { - super(); - this.client = client; - this.project_id = project_id; - this.init(); - } - - public get(): Info | undefined { - if (this.state != "ready") { - return; - } - const info = this.get_table()?.get(this.project_id)?.get("info"); - if (info == null) return; - return (info as never) as Info; - } - - public close(): void { - this.set_state("closed"); - if (this.table != null) { - this.table.close(); - } - this.removeAllListeners(); - close(this); - this.set_state("closed"); - } - - private async init(): Promise { - if (this.state != "init") { - throw Error("must be in init state"); - } - // Make sure there is a working websocket to the project - while (true) { - try { - await this.client.project_client.websocket(this.project_id); - break; - } catch (_) { - if (this.state == ("closed" as State)) return; - await delay(3000); - } - } - if ((this.state as State) == "closed") return; - - // Now create the table. - this.table = await this.client.sync_client.synctable_project( - this.project_id, - { - project_info: [ - { - project_id: this.project_id, - info: null, - }, - ], - }, - [{ ephemeral: true }] - ); - - if ((this.state as State) == "closed") return; - - this.table.on("change", (_: string[]) => { - this.emit("change"); - }); - this.set_state("ready"); - } - - // This is used to possibly work around a rare bug. - // https://github.com/sagemathinc/cocalc/issues/4790 - private async re_init(): Promise { - this.state = "init"; - await this.init(); - } - - private get_table(): SyncTable { - // TODO: some duplication with Listings -- would be nice to refactor (?). - if (this.state != "ready") { - throw Error("table not initialized "); - } - if (this.table == null) { - throw Error("table is null"); - } - if (this.table.get_state() == "closed") { - throw Error("table is closed"); - } - return this.table; - } - - public async set(info: Info | object): Promise { - if (!Map.isMap(info)) { - info = fromJS(info); - } - let table; - try { - table = this.get_table(); - } catch (err) { - // See https://github.com/sagemathinc/cocalc/issues/4790 - console.warn("Error getting table -- ", err); - await this.re_init(); - table = this.get_table(); - } - table.set({ project_id: this.project_id, info }, "shallow"); - await table.save(); - } - - public is_ready(): boolean { - return this.state == ("ready" as State); - } - - private set_state(state: State): void { - if (this.state == state) return; - if (this.state == "closed") { - throw Error("cannot switch away from closed"); - } - if (this.state == "ready" && state != "closed") { - throw Error("can only transition from ready to closed"); - } - this.state = state; - this.emit("state", state); - if (state === "ready") { - this.emit("ready"); - } - } -} - -export function project_info(client, project_id: string): ProjectInfo { - return new ProjectInfo(client, project_id); -} diff --git a/src/packages/frontend/project/websocket/project-status.ts b/src/packages/frontend/project/websocket/project-status.ts deleted file mode 100644 index 75ac35d776..0000000000 --- a/src/packages/frontend/project/websocket/project-status.ts +++ /dev/null @@ -1,155 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -/* -SyncTable of project status about a project. - -Use this via - - webapp_client.project_client.project_status(project_id) - -*/ - -import { EventEmitter } from "events"; -import { fromJS, Map } from "immutable"; -import { SyncTable } from "@cocalc/sync/table"; -import { delay } from "awaiting"; -import { close } from "@cocalc/util/misc"; -import { WebappClient } from "../../webapp-client"; - -type State = "init" | "ready" | "closed"; -type status = Map; - -export class ProjectStatus extends EventEmitter { - private table?: SyncTable; - private project_id: string; - private state: State = "init"; - private client: WebappClient; - - constructor(client: WebappClient, project_id: string) { - super(); - this.client = client; - this.project_id = project_id; - this.init(); - } - - public get(): status | undefined { - if (this.state != "ready") { - return; - } - const status = this.get_table()?.get(this.project_id)?.get("status"); - if (status == null) return; - return (status as never) as status; - } - - public close(): void { - this.set_state("closed"); - if (this.table != null) { - this.table.close(); - } - this.removeAllListeners(); - close(this); - this.set_state("closed"); - } - - private async init(): Promise { - if (this.state != "init") { - throw Error("must be in init state"); - } - // Make sure there is a working websocket to the project - while (true) { - try { - await this.client.project_client.websocket(this.project_id); - break; - } catch (_) { - if (this.state == ("closed" as State)) return; - await delay(3000); - } - } - if ((this.state as State) == "closed") return; - - // Now create the table. - this.table = await this.client.sync_client.synctable_project( - this.project_id, - { - project_status: [ - { - project_id: this.project_id, - status: null, - }, - ], - }, - [{ ephemeral: true }] - ); - - if ((this.state as State) == "closed") return; - - this.table.on("change", (_: string[]) => { - this.emit("change"); - }); - this.set_state("ready"); - } - - // This is used to possibly work around a rare bug. - // https://github.com/sagemathinc/cocalc/issues/4790 - private async re_init(): Promise { - this.state = "init"; - await this.init(); - } - - private get_table(): SyncTable { - // TODO: some duplication with Listings -- would be nice to refactor (?). - if (this.state != "ready") { - throw Error("table not initialized "); - } - if (this.table == null) { - throw Error("table is null"); - } - if (this.table.get_state() == "closed") { - throw Error("table is closed"); - } - return this.table; - } - - public async set(status: status | object): Promise { - if (!Map.isMap(status)) { - status = fromJS(status); - } - let table; - try { - table = this.get_table(); - } catch (err) { - // See https://github.com/sagemathinc/cocalc/issues/4790 - console.warn("Error getting table -- ", err); - await this.re_init(); - table = this.get_table(); - } - table.set({ project_id: this.project_id, status }, "shallow"); - await table.save(); - } - - public is_ready(): boolean { - return this.state == ("ready" as State); - } - - private set_state(state: State): void { - if (this.state == state) return; - if (this.state == "closed") { - throw Error("cannot switch away from closed"); - } - if (this.state == "ready" && state != "closed") { - throw Error("can only transition from ready to closed"); - } - this.state = state; - this.emit("state", state); - if (state === "ready") { - this.emit("ready"); - } - } -} - -export function project_status(client, project_id: string): ProjectStatus { - return new ProjectStatus(client, project_id); -} diff --git a/src/packages/frontend/project/websocket/usage-info.ts b/src/packages/frontend/project/websocket/usage-info.ts deleted file mode 100644 index 3ad1508885..0000000000 --- a/src/packages/frontend/project/websocket/usage-info.ts +++ /dev/null @@ -1,198 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -import { EventEmitter } from "events"; -import { delay } from "awaiting"; -import { SyncTable } from "@cocalc/sync/table"; -import { webapp_client } from "../../webapp-client"; -import { once } from "@cocalc/util/async-utils"; -import { merge } from "@cocalc/util/misc"; -import type { ImmutableUsageInfo } from "@cocalc/util/types/project-usage-info"; - -type State = "init" | "ready" | "closed"; - -export class UsageInfoWS extends EventEmitter { - private table?: SyncTable; - private readonly project_id: string; - private state: State = "init"; - private readonly last_version: { [path: string]: ImmutableUsageInfo } = {}; - - constructor(project_id: string) { - super(); - this.project_id = project_id; - this.init(); - } - - public event_key(path: string) { - return `path::${path}`; - } - - private async init(): Promise { - if (this.state != "init") { - throw Error("must be in init state"); - } - // Make sure there is a working websocket to the project - while (true) { - try { - await webapp_client.project_client.websocket(this.project_id); - break; - } catch (_) { - if (this.state == ("closed" as State)) return; - await delay(3000); - } - } - if ((this.state as State) == "closed") return; - - // Now create the table. - this.table = await webapp_client.sync_client.synctable_project( - this.project_id, - { - usage_info: [ - { - project_id: this.project_id, - path: null, - usage: null, - }, - ], - }, - [{ ephemeral: true }] - ); - - if ((this.state as State) == "closed") return; - - this.table.on("change", async (keys: string[]) => { - if (this.state != "ready") { - // don't do anything if being initialized or already closed, - // since code below will break in weird ways. - return; - } - // emit "real" changes of usage_info to interested parties - for (const key of keys) { - const path = JSON.parse(key)[1]; - // Be careful to only emit a change event if the actual - // usage itself changes. Table emits more frequently! - - const usage_record = this.get_record(path); - if (usage_record == null) continue; - const usage: ImmutableUsageInfo | undefined = usage_record.get("usage"); - //console.log(`UsageInfo table.on.change path='${path}' → usage=`, usage); - if (usage == null) continue; - if (usage != this.last_version[path]) { - this.last_version[path] = usage; - this.emit(this.event_key(path), usage); - } - } - }); - this.set_state("ready"); - } - - // copied from ./listings.ts - private async re_init(): Promise { - this.state = "init"; - await this.init(); - } - - private key(path: string): string { - return JSON.stringify([this.project_id, path]); - } - - private get_table(): SyncTable { - if (this.state != "ready") { - throw Error("table not initialized "); - } - if (this.table == null) { - throw Error("table is null"); - } - if (this.table.get_state() == "closed") { - throw Error("table is closed"); - } - return this.table; - } - - private get_record(path: string): ImmutableUsageInfo | undefined { - const x = this.get_table().get(this.key(path)); - if (x == null) return x; - return (x as unknown) as ImmutableUsageInfo; // coercing to fight typescript. - } - - private async get_table_safe(): Promise { - try { - return this.get_table(); - } catch (err) { - // See https://github.com/sagemathinc/cocalc/issues/4790 - console.warn("Error getting table -- ", err); - await this.re_init(); - return this.get_table(); - } - } - - private async set(obj: { path: string; usage?: any }): Promise { - const table = await this.get_table_safe(); - table.set(merge({ project_id: this.project_id }, obj), "shallow"); - await table.save(); - } - - private set_state(state: State): void { - if (this.state == state) return; - if (this.state == "closed") { - throw Error("cannot switch away from closed"); - } - if (this.state == "ready" && state != "closed") { - throw Error("can only transition from ready to closed"); - } - this.state = state; - this.emit("state", state); - } - - // we add the path we are interested in - public async watch(path: string): Promise { - // console.log(`UsageInfo watching ${this.project_id} / ${path}`); - if (await this.wait_until_ready(false)) return; - if (this.state == "closed") return; - this.set({ path }); - } - - // we remove the project/path key - public async disregard(path: string): Promise { - // console.log(`UsageInfo disregarding ${this.project_id} / ${path}`); - if (this.state == "closed") return; - const table = await this.get_table_safe(); - const data = table.get(); - if (data == null) return; - table.set(data.delete(this.key(path)), "none"); - await table.save(); - } - - // Returns true if never will be ready - private async wait_until_ready(exception: boolean = true): Promise { - try { - if (this.state == "closed") { - throw Error("UsageInfoWS object must not be closed"); - } - if (this.state == "init") { - await once(this, "state"); - if ((this.state as State) != "ready") { - throw Error("never will be ready"); - } - return false; - } - return false; - } catch (err) { - if (exception) throw err; - return true; - } - } -} - -// for each project, there is one instance -const usage_infos: { [project_id: string]: UsageInfoWS } = {}; - -export function get_usage_info(project_id: string): UsageInfoWS { - if (usage_infos[project_id] != null) { - return usage_infos[project_id]; - } else { - return (usage_infos[project_id] = new UsageInfoWS(project_id)); - } -} diff --git a/src/packages/frontend/project_actions.ts b/src/packages/frontend/project_actions.ts index 880260d7e1..0e612d241f 100644 --- a/src/packages/frontend/project_actions.ts +++ b/src/packages/frontend/project_actions.ts @@ -8,7 +8,7 @@ declare let window, document, $; import * as async from "async"; import { callback } from "awaiting"; -import { List, Map, Set, fromJS } from "immutable"; +import { List, Map, Set as immutableSet, fromJS } from "immutable"; import { isEqual, throttle } from "lodash"; import { join } from "path"; import { defineMessage } from "react-intl"; @@ -97,7 +97,7 @@ import { } from "@cocalc/frontend/project_store"; import track from "@cocalc/frontend/user-tracking"; import { webapp_client } from "@cocalc/frontend/webapp-client"; -import { retry_until_success } from "@cocalc/util/async-utils"; +import { once, retry_until_success } from "@cocalc/util/async-utils"; import { DEFAULT_NEW_FILENAMES, NEW_FILENAMES } from "@cocalc/util/db-schema"; import * as misc from "@cocalc/util/misc"; import { reduxNameToProjectId } from "@cocalc/util/redux/name"; @@ -107,7 +107,9 @@ import { get_editor } from "./editors/react-wrapper"; import { computeServerManager, type ComputeServerManager, -} from "@cocalc/nats/compute/manager"; +} from "@cocalc/conat/compute/manager"; +import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; +import { get as getProjectStatus } from "@cocalc/conat/project/project-status"; const { defaults, required } = misc; @@ -281,43 +283,120 @@ export const FILE_ACTIONS: { [key: string]: FileAction } = { } as const; export class ProjectActions extends Actions { + public state: "ready" | "closed" = "ready"; public project_id: string; private _last_history_state: string; private last_close_timer: number; - private _activity_indicator_timers: { [key: string]: number }; + private _activity_indicator_timers: { [key: string]: number } = {}; private _init_done = false; - private new_filename_generator; - public open_files?: OpenFiles; + private new_filename_generator = new NewFilenames("", false); private modal?: ModalInfo; + + // these are all potentially expensive + public open_files?: OpenFiles; private computeServerManager?: ComputeServerManager; + private projectStatusSub?; constructor(name, b) { super(name, b); this.project_id = reduxNameToProjectId(name); - this.new_filename_generator = new NewFilenames("", false); - this._activity_indicator_timers = {}; - this.open_files = new OpenFiles(this); - this.initNatsPermissions(); - this.initComputeServers(); - } + // console.log("create project actions", this.project_id); + // console.trace("create project actions", this.project_id) + this.expensiveLoop(); + } + + // COST -- there's a lot of code all over that may create project actions, + // e.g., when configuring a course with 150 students, then 150 project actions + // get created to do various operations. The big use of project actions + // though is when an actual tab is open in the UI with projects. + // So we put actions in two states: 'cheap' and 'expensive'. + // In the expensive state, there can be compute server changefeeds, + // etc. In the cheap state we close all that. When the tab is + // visibly open in the UI then expensive stuff automatically gets + // initialized, and when it is closed, it is destroyed. + + // actually open in the UI? + private lastProjectTabs: List = List([]); + private lastProjectTabOpenedState = false; + isTabOpened = () => { + const store = redux.getStore("projects"); + if (store == null) { + return false; + } + const projectTabs = store.get("open_projects") as List | undefined; + if (projectTabs == null) { + return false; + } + if (projectTabs.equals(this.lastProjectTabs)) { + return this.lastProjectTabOpenedState; + } + this.lastProjectTabs = projectTabs; + this.lastProjectTabOpenedState = projectTabs.includes(this.project_id); + return this.lastProjectTabOpenedState; + }; + isTabClosed = () => !this.isTabOpened(); - public async api(): Promise { - return await webapp_client.project_client.api(this.project_id); - } + private expensiveLoop = async () => { + while (this.state != "closed") { + if (this.isTabOpened()) { + this.initExpensive(); + } else { + this.closeExpensive(); + } + const store = redux.getStore("projects"); + if (store != null) { + await once(store, "change"); + } + } + }; - destroy = (): void => { - if (this.open_files == null) return; - this.closeComputeServers(); + private initialized = false; + private initExpensive = () => { + if (this.initialized) return; + // console.log("initExpensive", this.project_id); + this.initialized = true; + this.open_files = new OpenFiles(this); + this.initComputeServerManager(); + this.initComputeServersTable(); + this.initProjectStatus(); + const store = this.get_store(); + store?.init_table("public_paths"); + }; + + private closeExpensive = () => { + if (!this.initialized) return; + // console.log("closeExpensive", this.project_id); + this.initialized = false; + redux.removeProjectReferences(this.project_id); + this.closeComputeServerManager(); + this.closeComputeServerTable(); + this.projectStatusSub?.close(); + delete this.projectStatusSub; must_define(this.redux); this.close_all_files(); for (const table in QUERIES) { this.remove_table(table); } - this.open_files.close(); + + this.open_files?.close(); delete this.open_files; - this.computeServerManager?.close(); - delete this.computeServerManager; - webapp_client.nats_client.closeOpenFiles(this.project_id); + webapp_client.conat_client.closeOpenFiles(this.project_id); + + const store = this.get_store(); + store?.close_all_tables(); + }; + + public async api(): Promise { + return await webapp_client.project_client.api(this.project_id); + } + + destroy = (): void => { + // console.log("destroy project actions", this.project_id); + if (this.state == "closed") { + return; + } + this.closeExpensive(); + this.state = "closed"; }; private save_session(): void { @@ -974,6 +1053,8 @@ export class ProjectActions extends Actions { // Open the given file in this project. open_file = async (opts: OpenFileOpts): Promise => { + // Log that we *started* opening the file. + log_file_open(this.project_id, opts.path); await open_file(this, opts); }; @@ -1017,8 +1098,6 @@ export class ProjectActions extends Actions { ext, ); - // Log that we opened the file. - log_file_open(this.project_id, path); return { name, Editor }; }; @@ -1259,6 +1338,9 @@ export class ProjectActions extends Actions { } private touchActiveFileIfOnComputeServer = throttle(async (path: string) => { + if (this.state == "closed") { + return; + } const computeServerAssociations = webapp_client.project_client.computeServers(this.project_id); // this is what is currently configured: @@ -1304,7 +1386,7 @@ export class ProjectActions extends Actions { // Closes the file and removes all references. // Does not update tabs - close_file(path: string): void { + close_file = (path: string): void => { path = normalize(path); const store = this.get_store(); if (store == undefined) { @@ -1321,10 +1403,10 @@ export class ProjectActions extends Actions { component_data.is_public, ); this.save_session(); - } + }; // Makes this project the active project tab - foreground_project(change_history = true): void { + foreground_project = (change_history = true): void => { this._ensure_project_is_open((err) => { if (err) { // TODO! @@ -1593,7 +1675,7 @@ export class ProjectActions extends Actions { return; } const changes: { - checked_files?: Set; + checked_files?: immutableSet; file_action?: string | undefined; } = {}; if (checked) { @@ -1623,7 +1705,7 @@ export class ProjectActions extends Actions { return; } const changes: { - checked_files: Set; + checked_files: immutableSet; file_action?: string | undefined; } = { checked_files: store.get("checked_files").union(file_list) }; const file_action = store.get("file_action"); @@ -1645,7 +1727,7 @@ export class ProjectActions extends Actions { return; } const changes: { - checked_files: Set; + checked_files: immutableSet; file_action?: string | undefined; } = { checked_files: store.get("checked_files").subtract(file_list) }; @@ -1938,75 +2020,79 @@ export class ProjectActions extends Actions { // retrieve project configuration (capabilities, etc.) from the back-end // also return it as a convenience - async init_configuration( - aspect: ConfigurationAspect = "main", - no_cache = false, - ): Promise { - this.setState({ configuration_loading: true }); + init_configuration = reuseInFlight( + async ( + aspect: ConfigurationAspect = "main", + no_cache = false, + ): Promise => { + this.setState({ configuration_loading: true }); - const store = this.get_store(); - if (store == null) { - // console.warn("project_actions::init_configuration: no store"); - this.setState({ configuration_loading: false }); - return; - } + const store = this.get_store(); + if (store == null) { + // console.warn("project_actions::init_configuration: no store"); + this.setState({ configuration_loading: false }); + return; + } - const prev = store.get("configuration") as ProjectConfiguration; - if (!no_cache) { - // already done before? - if (prev != null) { - const conf = prev.get(aspect) as Configuration; - if (conf != null) { - this.setState({ configuration_loading: false }); - return conf; + const prev = store.get("configuration") as ProjectConfiguration; + if (!no_cache) { + // already done before? + if (prev != null) { + const conf = prev.get(aspect) as Configuration; + if (conf != null) { + this.setState({ configuration_loading: false }); + return conf; + } } } - } - // we do not know the configuration aspect. "next" will be the updated datastructure. - let next; + // we do not know the configuration aspect. "next" will be the updated datastructure. + let next; - await retry_until_success({ - f: async () => { - try { - next = await get_configuration( - webapp_client, - this.project_id, - aspect, - prev, - no_cache, - ); - } catch (e) { - // not implemented error happens, when the project is still the old one - // in that case, do as if everything is available - if (e.message.indexOf("not implemented") >= 0) { - return null; + await retry_until_success({ + f: async () => { + try { + next = await get_configuration( + webapp_client, + this.project_id, + aspect, + prev, + no_cache, + ); + } catch (e) { + // not implemented error happens, when the project is still the old one + // in that case, do as if everything is available + if (e.message.indexOf("not implemented") >= 0) { + return null; + } + // console.log( + // `WARNING -- project_actions::init_configuration err: ${e}`, + // ); + throw e; } - // console.log("project_actions::init_configuration err:", e); - throw e; - } - }, - start_delay: 1000, - max_delay: 5000, - desc: "project_actions::init_configuration", - }); + }, + start_delay: 2000, + max_delay: 5000, + desc: "project_actions::init_configuration", + }); - // there was a problem or configuration is not known - if (next == null) { - this.setState({ configuration_loading: false }); - return; - } + // there was a problem or configuration is not known + if (next == null) { + this.setState({ configuration_loading: false }); + return; + } - this.setState( - fromJS({ - configuration: next, - available_features: feature_is_available(next), - configuration_loading: false, - } as any), - ); + this.setState( + fromJS({ + configuration: next, + available_features: feature_is_available(next), + configuration_loading: false, + } as any), + ); - return next.get(aspect) as Configuration; - } + return next.get(aspect) as Configuration; + }, + ); // this is called once by the project initialization private async init_library() { @@ -2113,9 +2199,9 @@ export class ProjectActions extends Actions { misc.retry_until_success({ f: fetch, - start_delay: 1000, - max_delay: 10000, - max_time: 1000 * 60 * 3, // try for at most 3 minutes + start_delay: 15000, + max_delay: 30000, + max_time: 1000 * 60, // try for at most 3 minutes cb: () => { _init_library_index_ongoing[this.project_id] = false; }, @@ -2361,7 +2447,7 @@ export class ProjectActions extends Actions { opts0.target_path, misc.path_split(src_path).tail, ); - opts0.timeout = 90; + opts0.timeout = 90 * 1000; try { await webapp_client.project_client.copy_path_between_projects(opts0); cb(); @@ -3038,38 +3124,6 @@ export class ProjectActions extends Actions { } } - private async neuralSearch(text, path) { - try { - const scope = `projects/${this.project_id}/files/${path}`; - const results = await webapp_client.openai_client.embeddings_search({ - text, - limit: 25, - scope, - }); - const search_results: { - filename: string; - description: string; - fragment_id?: FragmentId; - }[] = []; - for (const result of results) { - const url = result.payload["url"] as string | undefined; - if (!url) continue; - const [filename, fragment_id] = url.slice(scope.length + 1).split("#"); - const description = result.payload["text"] ?? ""; - search_results.push({ - filename: filename[0] == "/" ? filename.slice(1) : filename, - description, - fragment_id: Fragment.decode(fragment_id), - }); - } - this.setState({ search_results }); - } catch (err) { - this.setState({ - search_error: `${err}`, - }); - } - } - search = () => { let cmd, ins; const store = this.get_store(); @@ -3101,11 +3155,6 @@ export class ProjectActions extends Actions { git_grep: store.get("git_grep"), }); - if (store.get("neural_search")) { - this.neuralSearch(query, path); - return; - } - // generate the grep command for the given query with the given flags if (store.get("case_sensitive")) { ins = ""; @@ -3504,7 +3553,7 @@ export class ProjectActions extends Actions { this.setRecentlyDeleted(path, 0); (async () => { try { - const o = await webapp_client.nats_client.openFiles(this.project_id); + const o = await webapp_client.conat_client.openFiles(this.project_id); o.setNotDeleted(path); } catch (err) { console.log("WARNING: issue undeleting file", err); @@ -3512,19 +3561,43 @@ export class ProjectActions extends Actions { })(); }; - private initComputeServers = () => { + private initProjectStatus = async () => { + this.projectStatusSub = await getProjectStatus({ + project_id: this.project_id, + compute_server_id: 0, + }); + for await (const mesg of this.projectStatusSub) { + const status = mesg.data; + this.setState({ status }); + } + }; + + private initComputeServersTable = () => { // table of information about all the compute servers in this project computeServers.init(this.project_id); + }; + + private closeComputeServerTable = () => { + computeServers.close(this.project_id); + }; + + private initComputeServerManager = () => { + // console.log("initComputeServerManager"); + if (this.state == "closed") { + return; + } // table mapping paths to the id of the compute server it is hosted on this.computeServerManager = computeServerManager({ project_id: this.project_id, }); - this.computeServerManager.on("connected", () => { - if (this.computeServerManager == null) { + this.computeServerManager.once("connected", () => { + if (this.state == "closed" || this.computeServerManager == null) { return; } - const compute_server_ids = this.computeServerManager.getAll() as any; - for (let path in compute_server_ids) { + const compute_server_ids = { + ...this.computeServerManager.getAll(), + } as any; + for (const path in compute_server_ids) { compute_server_ids[path] = compute_server_ids[path].id; } this.setState({ compute_server_ids }); @@ -3535,23 +3608,16 @@ export class ProjectActions extends Actions { ); }; - private initNatsPermissions = async () => { - try { - await webapp_client.nats_client.addProjectPermissions([this.project_id]); - } catch (err) { - console.log( - `WARNING: issue getting permission to access project ${this.project_id} -- ${err}`, - ); + private closeComputeServerManager = () => { + // console.log("closeComputeServerManager"); + if (this.computeServerManager == null) { + return; } - }; - - private closeComputeServers = () => { - computeServers.close(this.project_id); - this.computeServerManager?.removeListener( + this.computeServerManager.removeListener( "change", this.handleComputeServerManagerChange, ); - this.computeServerManager?.close(); + this.computeServerManager.close(); delete this.computeServerManager; }; @@ -3644,7 +3710,7 @@ export class ProjectActions extends Actions { }; projectApi = (opts?) => { - return webapp_client.nats_client.projectApi({ + return webapp_client.conat_client.projectApi({ ...opts, project_id: this.project_id, }); diff --git a/src/packages/frontend/project_configuration.ts b/src/packages/frontend/project_configuration.ts index 3ee6b14234..37aea5beb9 100644 --- a/src/packages/frontend/project_configuration.ts +++ b/src/packages/frontend/project_configuration.ts @@ -60,11 +60,7 @@ export function isMainConfiguration( ): config is MainConfiguration { const mconf = config; // don't test for disabled_ext, because that's added later - return ( - isMainCapabilities(mconf.capabilities) && - mconf.timestamp != null && - typeof mconf.timestamp == "string" - ); + return isMainCapabilities(mconf.capabilities) && !!mconf.timestamp; } // if prettier exists, this adds all syntaxes to format via prettier @@ -165,11 +161,16 @@ export async function get_configuration( aspect, no_cache, ); - if (config == null) return prev; + if (config == null) { + return prev; + } // console.log("project_actions::init_configuration", aspect, config); if (aspect == ("main" as ConfigurationAspect)) { - if (!isMainConfiguration(config)) return; + if (!isMainConfiguration(config)) { + console.log("reject", isMainConfiguration(config), config); + return; + } const caps = config.capabilities; // TEST x11/latex/sage disabilities // caps.x11 = false; @@ -184,7 +185,9 @@ export async function get_configuration( // jupyter.lab = false; // TEST no kernelspec → we can't read any kernels → entirely disable jupyter // jupyter.kernelspec = false; - if (!jupyter.kernelspec) caps.jupyter = false; + if (!jupyter.kernelspec) { + caps.jupyter = false; + } } // disable/hide certain file extensions if certain capabilities are missing diff --git a/src/packages/frontend/project_store.ts b/src/packages/frontend/project_store.ts index db7e4d711b..3b7b4416a9 100644 --- a/src/packages/frontend/project_store.ts +++ b/src/packages/frontend/project_store.ts @@ -26,7 +26,7 @@ import { TypedMap, } from "@cocalc/frontend/app-framework"; import { ProjectLogMap } from "@cocalc/frontend/project/history/types"; -import { Listings, listings } from "@cocalc/frontend/nats/listings"; +import { Listings, listings } from "@cocalc/frontend/conat/listings"; import { FILE_ACTIONS, ProjectActions, @@ -56,6 +56,7 @@ import { import { get_local_storage } from "@cocalc/frontend/misc"; import { QueryParams } from "@cocalc/frontend/misc/query-params"; import { fileURL } from "@cocalc/frontend/lib/cocalc-urls"; +import { remove } from "@cocalc/frontend/project-file"; export { FILE_ACTIONS as file_actions, ProjectActions }; @@ -190,6 +191,8 @@ export class ProjectStore extends Store { // is a little awkward, since I didn't want to change things too // much while making this optimization. public init_table: (table_name: string) => void; + public close_table: (table_name: string) => void; + public close_all_tables: () => void; // name = 'project-[project-id]' = name of the store // redux = global redux object @@ -228,6 +231,13 @@ export class ProjectStore extends Store { this.listings[id].close(); delete this.listings[id]; } + // close any open file tabs, properly cleaning up editor state: + const open = this.get("open_files")?.toJS(); + if (open != null) { + for (const path in open) { + remove(path, redux, this.project_id, false); + } + } }; // constructor binds this callback, such that "this.project_id" works! @@ -746,6 +756,7 @@ export function init(project_id: string, redux: AppRedux): ProjectStore { store._init(); const queries = misc.deep_copy(QUERIES); + const create_table = function (table_name, q) { //console.log("create_table", table_name) return class P extends Table { @@ -769,9 +780,14 @@ export function init(project_id: string, redux: AppRedux): ProjectStore { }; function init_table(table_name: string): void { + const name = project_redux_name(project_id, table_name); + try { + // throws error only if it does not exist already + redux.getTable(name); + return; + } catch {} + const q = queries[table_name]; - if (q == null) return; // already done - delete queries[table_name]; // so we do not init again. for (const k in q) { const v = q[k]; if (typeof v === "function") { @@ -779,18 +795,20 @@ export function init(project_id: string, redux: AppRedux): ProjectStore { } } q.query.project_id = project_id; - redux.createTable( - project_redux_name(project_id, table_name), - create_table(table_name, q), - ); + redux.createTable(name, create_table(table_name, q)); } - // public_paths is needed to show file listing and show - // any individual file, so we just load it... - init_table("public_paths"); - // project_log, on the other hand, is only loaded if needed. - store.init_table = init_table; + store.close_table = (table_name: string) => { + redux.removeTable(project_redux_name(project_id, table_name)); + }; + + store.close_all_tables = () => { + for (const table_name in queries) { + store.close_table(table_name); + } + }; + return store; } diff --git a/src/packages/frontend/projects/actions.ts b/src/packages/frontend/projects/actions.ts index 5e1452814a..5d7167b9c8 100644 --- a/src/packages/frontend/projects/actions.ts +++ b/src/packages/frontend/projects/actions.ts @@ -58,13 +58,13 @@ export class ProjectsActions extends Actions { return the_table; }; - private async projects_table_set( + private projects_table_set = async ( obj: object, merge: "deep" | "shallow" | "none" | undefined = "deep", - ): Promise { + ): Promise => { const table = await this.getProjectTable(); await table?.set(obj, merge); - } + }; // Set something in the projects table of the database directly // using a query, instead of using sync'd table mechanism, which @@ -688,7 +688,7 @@ export class ProjectsActions extends Actions { const email = markdown_to_html(body); try { - const resp = await webapp_client.project_collaborators.invite_noncloud({ + await webapp_client.project_collaborators.invite_noncloud({ project_id, title, link2proj, @@ -699,7 +699,9 @@ export class ProjectsActions extends Actions { subject, }); if (!silent) { - alert_message({ message: resp.mesg }); + alert_message({ + message: `Invited ${to} to collaborate on project.`, + }); } } catch (err) { if (!silent) { diff --git a/src/packages/frontend/projects/project-list-desc.tsx b/src/packages/frontend/projects/project-list-desc.tsx index 45a03ebaaa..7797129876 100644 --- a/src/packages/frontend/projects/project-list-desc.tsx +++ b/src/packages/frontend/projects/project-list-desc.tsx @@ -5,9 +5,6 @@ import { Button } from "antd"; import { useIntl } from "react-intl"; - -import { ResetProjectsConfirmation } from "@cocalc/frontend/account/upgrades/reset-projects"; -import { alert_message } from "@cocalc/frontend/alerts"; import { Alert, ButtonGroup, @@ -22,7 +19,6 @@ import { } from "@cocalc/frontend/app-framework"; import { Gap, Icon } from "@cocalc/frontend/components"; import { labels } from "@cocalc/frontend/i18n"; -import { webapp_client } from "@cocalc/frontend/webapp-client"; import { plural } from "@cocalc/util/misc"; interface Props { @@ -98,18 +94,12 @@ export const ProjectsListingDescription: React.FC = ({ function render_projects_actions_toolbar(): JSX.Element { return ( - {visible_projects.length > 0 - ? render_remove_from_all_button() - : undefined} {visible_projects.length > 0 && !deleted ? render_delete_all_button() : undefined} {visible_projects.length > 0 && !hidden ? render_hide_all_button() : undefined} - {visible_projects.length > 0 - ? render_remove_upgrades_from_all_button() - : undefined} {visible_projects.length > 0 ? render_stop_all_button() : undefined} {visible_projects.length > 0 ? render_restart_all_button() : undefined} @@ -120,10 +110,6 @@ export const ProjectsListingDescription: React.FC = ({ switch (show_alert) { case "hide": return render_hide_all(); - case "remove": - return render_remove_from_all(); - case "remove-upgrades": - return render_remove_upgrades_from_all(); case "delete": return render_delete_all(); case "stop": @@ -187,28 +173,6 @@ export const ProjectsListingDescription: React.FC = ({ ); } - function render_remove_from_all_button(): JSX.Element { - return ( - - ); - } - - function render_remove_upgrades_from_all_button(): JSX.Element { - return ( - - ); - } - function render_stop_all_button(): JSX.Element { return ( - - ); - } else { - let desc; - if (v.length < visible_projects.length) { - const other = visible_projects.length - v.length; - desc = `You are a collaborator on ${v.length} of the ${ - visible_projects.length - } ${plural( - visible_projects.length, - "project", - )} listed here (you own the other ${plural(other, "one")}).`; - } else { - if (v.length === 1) { - desc = "You are a collaborator on the one project listed here."; - } else { - desc = `You are a collaborator on ALL of the ${v.length} ${plural( - v.length, - "project", - )} listed here.`; - } - } - return ( - - {head} {desc} -

    - Are you sure you want to remove yourself from the {v.length}{" "} - {plural(v.length, "project")} listed below that you collaborate on? -
    - - You will no longer have access and cannot add yourself back. - {" "} - - - - - - ); - } - } - - function do_remove_from_all(): void { - for (const project_id of collab_projects()) { - actions.remove_collaborator(project_id, account_id); - } - set_show_alert("none"); - } - function render_can_be_undone(): JSX.Element { return ( diff --git a/src/packages/frontend/sagews/sagews.coffee b/src/packages/frontend/sagews/sagews.coffee index 2d36975269..96bb5ddacd 100644 --- a/src/packages/frontend/sagews/sagews.coffee +++ b/src/packages/frontend/sagews/sagews.coffee @@ -1159,6 +1159,7 @@ class SynchronizedWorksheet extends SynchronizedDocument2 y.attr('src', new_src) _post_save_success: () => + console.log("_post_save_success") @remove_output_blob_ttls() # Return array of uuid's of blobs that might possibly be in the worksheet @@ -1184,10 +1185,12 @@ class SynchronizedWorksheet extends SynchronizedDocument2 # TODO: prioritize automatic testing of this highly... since it is easy to break by changing # how worksheets render slightly. uuids = @_output_blobs_with_possible_ttl() + console.log("remove_output_blob_ttls -- ", uuids) if uuids? try - await webapp_client.file_client.remove_blob_ttls(uuids) + await webapp_client.conat_client.hub.db.removeBlobTtls({uuids:uuids}) catch err + console.log("WARNING: problem removing ttl from sage worksheet blobs ", err) cb?(err) return # don't try again to remove ttls for these blobs -- since did so successfully diff --git a/src/packages/frontend/site-licenses/purchase/index.ts b/src/packages/frontend/site-licenses/purchase/index.ts deleted file mode 100644 index 52b54c5269..0000000000 --- a/src/packages/frontend/site-licenses/purchase/index.ts +++ /dev/null @@ -1,7 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -export { PurchaseOneLicense } from "./purchase"; -export { PurchaseOneLicenseLink } from "./link"; \ No newline at end of file diff --git a/src/packages/frontend/site-licenses/purchase/link.tsx b/src/packages/frontend/site-licenses/purchase/link.tsx deleted file mode 100644 index 9d1dfb495a..0000000000 --- a/src/packages/frontend/site-licenses/purchase/link.tsx +++ /dev/null @@ -1,39 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -/* Link to purchasing a license */ - -import { redux, useTypedRedux } from "@cocalc/frontend/app-framework"; -import { Icon, Gap } from "@cocalc/frontend/components"; -import { Button } from "antd"; -import { PurchaseOneLicense } from "./purchase"; - -export const PurchaseOneLicenseLink: React.FC = () => { - const expand = useTypedRedux("account", "show_purchase_form") ?? false; - - function set_expand(show: boolean) { - redux.getActions("account").set_show_purchase_form(show); - } - - return ( -

    - - {expand && ( - <> -
    -
    - { - set_expand(false); - }} - /> - - )} -
    - ); -}; diff --git a/src/packages/frontend/site-licenses/purchase/purchase-method.tsx b/src/packages/frontend/site-licenses/purchase/purchase-method.tsx deleted file mode 100644 index 0076268420..0000000000 --- a/src/packages/frontend/site-licenses/purchase/purchase-method.tsx +++ /dev/null @@ -1,90 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -/* Component that obtains a payment method for this user: - - - if one is already available, user can confirm that - - if no payment method is available, they enter one - - onClose is called with the method or null if user decides not to enter a method. - -*/ -import { Button } from "antd"; - -import { - React, - useActions, - useAsyncEffect, - useTypedRedux, - useState, -} from "@cocalc/frontend/app-framework"; -import { Icon, Loading, Gap } from "@cocalc/frontend/components"; -import { alert_message } from "@cocalc/frontend/alerts"; -import { PaymentMethods } from "@cocalc/frontend/billing/payment-methods"; - -interface Props { - onClose: (id: string | undefined) => void; - amount: string; // amount formated as a currency - description: string; -} - -export const PurchaseMethod: React.FC = React.memo( - ({ amount, description, onClose }) => { - const customer = useTypedRedux("billing", "customer"); - const actions = useActions("billing"); - const [loaded, set_loaded] = useState(false); - const [buy_confirm, set_buy_confirm] = useState(false); - - useAsyncEffect(async (isMounted) => { - // update billing info whenever component mounts - try { - await actions.update_customer(); - } catch (err) { - alert_message({ - type: "error", - message: `Problem loading customer info -- ${err}`, - }); - } - if (isMounted()) { - set_loaded(true); - } - }, []); - - if (!loaded) { - return ; - } - - const source = customer?.get("default_source"); - return ( -
    - - {source && ( - - )} - {source && buy_confirm && ( -
    -
    - Charge the default card {amount} plus any applicable tax for{" "} - {description}. -
    -
    - -
    - )} -
    - ); - } -); diff --git a/src/packages/frontend/site-licenses/purchase/purchase.tsx b/src/packages/frontend/site-licenses/purchase/purchase.tsx deleted file mode 100644 index cc78b24bed..0000000000 --- a/src/packages/frontend/site-licenses/purchase/purchase.tsx +++ /dev/null @@ -1,791 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -/* - -Purchasing a license in the frontend app. - -DEPRECATED -- this is no longer used anywhere in the frontend code base. -Instead, use the nextjs app... unless we want to bring back frontend -purchases of licenses (maybe we do!) and then maybe this will be useful. - -*/ - -import { DownOutlined } from "@ant-design/icons"; -import { - CSS, - React, - redux, - useMemo, - useState, -} from "@cocalc/frontend/app-framework"; -import { DOC_LICENSE_URL } from "@cocalc/frontend/billing/data"; -import { - A, - CopyToClipBoard, - ErrorDisplay, - Icon, - Loading, - Gap, -} from "@cocalc/frontend/components"; -import { appBasePath } from "@cocalc/frontend/customize/app-base-path"; -import { supportURL } from "@cocalc/frontend/support/url"; -import { webapp_client } from "@cocalc/frontend/webapp-client"; -import { LicenseIdleTimeouts } from "@cocalc/util/consts/site-license"; -import { describe_quota } from "@cocalc/util/licenses/describe-quota"; -import { - COSTS, - discount_monthly_pct, - discount_pct, - discount_yearly_pct, -} from "@cocalc/util/licenses/purchase/consts"; -import { compute_cost } from "@cocalc/util/licenses/purchase/compute-cost"; -import type { - Cost, - PurchaseInfo, - Subscription, - Upgrade, - User, -} from "@cocalc/util/licenses/purchase/types"; -import { money } from "@cocalc/util/licenses/purchase/utils"; -import { plural } from "@cocalc/util/misc"; -import { endOfDay, getDays, startOfDay } from "@cocalc/util/stripe/timecalcs"; -import { COLORS } from "@cocalc/util/theme"; -import { - Button, - Card, - Col, - DatePicker, - Dropdown, - Input, - InputNumber, - Menu, - Row, -} from "antd"; -import dayjs from "dayjs"; -import { join } from "path"; -import { DebounceInput } from "react-debounce-input"; -import { create_quote_support_ticket } from "./get-a-quote"; -import { PurchaseMethod } from "./purchase-method"; -import { QuotaEditor } from "./quota-editor"; -import { RadioGroup } from "./radio-group"; -import { CURRENT_VERSION } from "@cocalc/util/licenses/purchase/consts"; - -const { RangePicker } = DatePicker; - -const LENGTH_PRESETS = [ - { label: "2 Days", desc: { n: 2, key: "days" } }, - { label: "1 Week", desc: { n: 7, key: "days" } }, - { label: "1 Month", desc: { n: 1, key: "months" } }, - { label: "6 Weeks", desc: { n: 7 * 6, key: "days" } }, - { label: "2 Months", desc: { n: 2, key: "months" } }, - { label: "3 Months", desc: { n: 3, key: "months" } }, - { label: "4 Months", desc: { n: 4, key: "months" } }, - { label: "5 Months", desc: { n: 5, key: "months" } }, - { label: "6 Months", desc: { n: 6, key: "months" } }, - { label: "7 Months", desc: { n: 7, key: "months" } }, - { label: "8 Months", desc: { n: 8, key: "months" } }, - { label: "9 Months", desc: { n: 9, key: "months" } }, - { label: "10 Months", desc: { n: 10, key: "months" } }, - { label: "11 Months", desc: { n: 11, key: "months" } }, - { label: "1 Year", desc: { n: 1, key: "years" } }, -] as const; - -const radioStyle: CSS = { - display: "block", - whiteSpace: "normal", - fontWeight: "inherit", // this is to undo what react-bootstrap does to the labels. -} as const; - -interface Props { - onClose: () => void; -} - -export const PurchaseOneLicense: React.FC = React.memo(({ onClose }) => { - const [user, set_user] = useState(undefined); - const [upgrade] = useState("custom"); - const [title, set_title] = useState(""); - const [description, set_description] = useState(""); - - const [custom_ram, set_custom_ram] = useState(COSTS.basic.ram); - const [custom_cpu, set_custom_cpu] = useState(COSTS.basic.cpu); - const [custom_dedicated_ram, set_custom_dedicated_ram] = useState( - COSTS.basic.dedicated_ram, - ); - const [custom_dedicated_cpu, set_custom_dedicated_cpu] = useState( - COSTS.basic.dedicated_cpu, - ); - const [custom_disk, set_custom_disk] = useState(COSTS.basic.disk); - const [custom_always_running, set_custom_always_running] = useState( - !!COSTS.basic.always_running, - ); - const [custom_member, set_custom_member] = useState( - !!COSTS.basic.member, - ); - const [custom_idle_timeout, set_custom_idle_timeout] = - useState("short"); - const [quantity, set_quantity] = useState(1); - const [subscription, set_subscription] = useState("monthly"); - - const [start, set_start_state] = useState(new Date()); - - function set_start(date: Date) { - date = date < start ? new Date() : date; - // start at midnight (local user time) on that day - date = startOfDay(date); - set_start_state(date); - } - - const [end, set_end_state] = useState(dayjs().add(1, "month").toDate()); - - function set_end(date: Date) { - const today = endOfDay(date); - const two_years = dayjs(start).add(2, "year").toDate(); - if (date <= today) { - date = today; - } else if (date >= two_years) { - date = two_years; - } - // ends at the last moment (local user time) for the user on that day - date = endOfDay(date); - set_end_state(date); - } - - const [quote, set_quote] = useState(undefined); - const [quote_info, set_quote_info] = useState(undefined); - const [error, set_error] = useState(""); - const [sending, set_sending] = useState< - undefined | "active" | "success" | "failed" - >(undefined); - const [purchase_resp, set_purchase_resp] = useState( - undefined, - ); - const disabled: boolean = useMemo(() => { - return sending == "success" || sending == "active"; - }, [sending]); - const [payment_method, set_payment_method] = useState( - undefined, - ); - - const cost = useMemo(() => { - if (user == null || quantity == null || subscription == null) { - return undefined; - } - return compute_cost({ - version: CURRENT_VERSION, - type: "quota", - quantity, - user, - upgrade, - subscription, - start, - end, - custom_ram, - custom_cpu, - custom_dedicated_ram, - custom_dedicated_cpu, - custom_disk, - custom_member, - custom_uptime: - custom_always_running == true ? "always_running" : custom_idle_timeout, - }); - }, [ - quantity, - user, - upgrade, - subscription, - start, - end, - custom_ram, - custom_cpu, - custom_dedicated_ram, - custom_dedicated_cpu, - custom_disk, - custom_always_running, - custom_member, - custom_idle_timeout, - ]); - - function render_error() { - if (error == "") return; - return ( - set_error("")} - /> - ); - } - - function render_user() { - return ( -
    -

    - Discount -

    - - students, teachers, academic researchers, non-profit - organizations and hobbyists ({discount_pct}% discount) - - ), - value: "academic", - icon: "graduation-cap", - }, - { - label: "Business", - desc: "for commercial business purposes", - value: "business", - icon: "briefcase", - }, - ]} - onChange={(e) => set_user(e.target.value)} - value={user} - disabled={disabled} - radioStyle={radioStyle} - /> -
    - ); - } - - function render_project_type() { - if (user == null || cost == null) return; - - return ( -
    -

    - Type - {`: ${money(cost.cost_per_project_per_month)}/month per project`} -

    -
    - Up to {quantity} projects can be running at once, each with the - following specs: -
    -
    - {render_custom()} -
    - ); - } - - function render_custom() { - if (user == null) return; - return ( - { - if (change.cpu != null) set_custom_cpu(change.cpu); - if (change.ram != null) set_custom_ram(change.ram); - if (change.dedicated_cpu != null) - set_custom_dedicated_cpu(change.dedicated_cpu); - if (change.dedicated_ram != null) - set_custom_dedicated_ram(change.dedicated_ram); - if (change.disk != null) set_custom_disk(change.disk); - if (change.member != null) set_custom_member(change.member); - if (change.always_running != null) - set_custom_always_running(change.always_running); - if (change.idle_timeout != null) - set_custom_idle_timeout(change.idle_timeout); - }} - /> - ); - } - - function render_quantity_input() { - return ( - { - if (typeof x != "number") return; - set_quantity(Math.round(x)); - }} - /> - ); - } - - function render_quantity() { - if (user == null) return; - return ( -
    -
    -

    - Number of Projects:{" "} - {render_quantity_input()} -

    -
    -
      -
    • - Simultaneously run {quantity} {plural(quantity, "project")} with - this license. You, and anyone you share the license code with, can - apply the license to any number of projects (in project settings). -
      • -
    • - {" "} - If you're{" "} - - teaching a course - - , the number of projects is typically n+2, where n{" "} - is the number of students in the class: each student has a - project, you will manage the course from a project, and all - students will have access to one shared project. Contact us by - clicking the "Help" button if you need to change the quantity - later in the course as more students add. -
      • -
    • - {" "} - You can create hundreds of projects that use this license, but - only {quantity} can be running at once. -
      • -
    -
    -
    - ); - } - - function render_subscription() { - if (user == null) return; - return ( -
    -
    -

    - Period -

    - set_subscription(e.target.value)} - value={subscription} - radioStyle={radioStyle} - /> -
    - ); - } - - function set_end_date(x): void { - set_end( - dayjs(start) - .subtract(1, "day") - .add(x.n as any, x.key) - .toDate(), - ); - } - - function render_date() { - if ( - upgrade == null || - user == null || - quantity == null || - subscription != "no" - ) - return; - // range of dates: start date -- end date - // TODO: use "midnight UTC", or should we just give a - // day grace period on both ends (?). - const value = [dayjs(start), dayjs(end)]; - const presets: JSX.Element[] = []; - for (const { label, desc } of LENGTH_PRESETS) { - presets.push( - - set_end_date(desc)}>{label} - , - ); - } - const menu = {presets}; - // this is fine, since getDays rounds the days difference, and start/end is set to the start/end of the day already - const n = getDays({ start, end }); - return ( -
    -
    -
    - Start and end dates ({n} {plural(n, "day")}) -
    - { - if (value == null || value[0] == null || value[1] == null) return; - set_start(value[0].toDate()); - set_end(value[1].toDate()); - }} - /> - - - - - e.preventDefault()}> - End after... - - -
    - ); - } - - function render_title_desc() { - if (cost == null) return; - return ( -
    -
    -

    - Title and description -

    - Optionally set the title and description of this license. You can easily - change this later. -
    -
    - - Title - - set_title(e.target.value)} - /> - - - Description - - set_description(e.target.value)} - /> - - -
    - ); - } - - function render_cost() { - if (cost == null) return; - const desc = `${money(cost.cost)} ${ - subscription != "no" ? subscription : "" - }`; - - return ( -
    -
    -

    - Cost: {desc} -

    -
    - ); - } - - function render_quote() { - if (cost == null) return; - return ( -
    -
    -

    - Purchase -

    - set_quote(e.target.value)} - value={quote} - radioStyle={radioStyle} - /> -
    - ); - } - - function render_credit_card() { - if (quote !== false || cost == null) return; - if (payment_method != null) { - // payment method already selected, which is only the case - // during payment and once it is done. - return; - } else { - // ask them to confirm their method and pay. - return ( -
    -
    -

    - Payment -

    - { - set_payment_method(id); - submit(); - }} - /> -
    - ); - } - } - - async function submit(): Promise { - if ( - user == null || - upgrade == null || - quantity == undefined || - subscription == null || - quote == null - ) - return; - const info: PurchaseInfo = { - version: CURRENT_VERSION, - type: "quota", - quantity, - user, - upgrade, - subscription, - start, - end, - quote, - quote_info, - payment_method, - cost, - custom_ram, - custom_cpu, - custom_dedicated_ram, - custom_dedicated_cpu, - custom_disk, - custom_member, - custom_uptime: - custom_always_running == true ? "always_running" : custom_idle_timeout, - title, - description, - }; - set_error(""); - if (quote) { - set_sending("success"); - create_quote_support_ticket(info); - onClose(); - } else { - set_sending("active"); - try { - const resp = await webapp_client.stripe.purchase_license(info); - set_purchase_resp(resp); - set_sending("success"); - redux.getActions("billing").update_managed_licenses(); - } catch (err) { - set_error(err.toString()); - set_sending("failed"); - } - } - } - - function render_quote_info() { - if (quote !== true) return; - - return ( -
    - Enter additional information about your quote request: -
    - set_quote_info(event.target.value)} - /> -
    -
    -
    - Click the button below to enter the above information in a support - request. We will then respond with more information. -
    -
    - -
    - ); - } - - function render_sending() { - switch (sending) { - case "active": - return ( -
    -

    - -

    -
    - ); - case "success": - return ( -
    -

    - Successfully{" "} - {quote === true - ? "requested quote; we will be in touch soon" - : "completed purchase"} - ! -

    -
    - ); - case "failed": - if (error) { - return ( -
    -

    - Failed to{" "} - {quote === true ? "request quote" : "complete purchase"}. Please - try again later. -

    -
    - ); - } else return; - } - } - - function render_purchase_resp() { - if (!purchase_resp) return; - return ( -
    - Your newly purchased license code is -
    -
    - - You should see it listed under{" "} - - Licenses You Manage - - . -
    - ); - } - - // Just cancel everything or close the dialog (since you're done). - function render_close() { - return ( -
    - -
    - ); - } - - function render_instructions() { - return ( -
    - Buy licenses or request a quote below, as{" "} - explained here, or{" "} - visit the new store. If you are - planning on making a significant purchase, but need to test things out - first,{" "} - - create a support ticket and request more details or a free trial - - . -
    - ); - } - - return ( - -

    Buy a license

    - - } - extra={close} - style={{ borderColor: COLORS.BLUE, maxWidth: "900px", margin: "auto" }} - > - {render_instructions()} - {render_user()} - {render_quantity()} - {render_project_type()} - {render_subscription()} - {render_date()} - {render_title_desc()} - {render_cost()} - {render_quote()} - {render_credit_card()} - {render_quote_info()} - {render_sending()} - {render_error()} - {render_purchase_resp()} -
    - {render_close()} -     - ); -}); diff --git a/src/packages/frontend/site-licenses/site-license-public-info.tsx b/src/packages/frontend/site-licenses/site-license-public-info.tsx index e4ef896cb7..253eab0419 100644 --- a/src/packages/frontend/site-licenses/site-license-public-info.tsx +++ b/src/packages/frontend/site-licenses/site-license-public-info.tsx @@ -9,6 +9,7 @@ import { Alert, Button, Popconfirm, Popover, Table, Tag, Tooltip } from "antd"; import { isEqual } from "lodash"; import { ReactNode } from "react"; import { useIntl } from "react-intl"; +import { isValidUUID } from "@cocalc/util/misc"; import { React, @@ -127,9 +128,12 @@ export const SiteLicensePublicInfoTable: React.FC = ( await Promise.all( Object.keys(site_licenses).map(async function (license_id) { try { + if(!isValidUUID(license_id)) { + return; + } const info = await site_license_public_info(license_id, force); if (info == null) { - throw new Error(`license ${license_id} not found`); + throw new Error(`license '${license_id}' not found`); } infos[license_id] = info; } catch (err) { @@ -232,8 +236,8 @@ export const SiteLicensePublicInfoTable: React.FC = ( status === "expired" ? true : v?.expires != null - ? new Date() >= v.expires - : false; + ? new Date() >= v.expires + : false; return { key: idx, license_id: k, diff --git a/src/packages/frontend/test/setup.js b/src/packages/frontend/test/setup.js new file mode 100644 index 0000000000..a1f29b8e50 --- /dev/null +++ b/src/packages/frontend/test/setup.js @@ -0,0 +1,3 @@ +const Enzyme = require("enzyme"); +const Adapter = require("@cfaester/enzyme-adapter-react-18").default; +Enzyme.configure({ adapter: new Adapter() }); diff --git a/src/packages/frontend/tsconfig.json b/src/packages/frontend/tsconfig.json index c91da6e99e..11043dc262 100644 --- a/src/packages/frontend/tsconfig.json +++ b/src/packages/frontend/tsconfig.json @@ -28,7 +28,7 @@ "references": [ { "path": "../comm" }, { "path": "../jupyter" }, - { "path": "../nats" }, + { "path": "../conat" }, { "path": "../sync" }, { "path": "../util" } ] diff --git a/src/packages/frontend/webapp-hooks.ts b/src/packages/frontend/webapp-hooks.ts index 733ec7b272..da90425183 100644 --- a/src/packages/frontend/webapp-hooks.ts +++ b/src/packages/frontend/webapp-hooks.ts @@ -31,22 +31,11 @@ export function init() { cb(); }; - webapp_client.on("mesg_info", function (info) { - const f = () => { - const account_store = redux.getActions("account"); - if (account_store != undefined) { - account_store.setState({ mesg_info: info }); - } - }; - // must be scheduled separately, since this notification can be triggered during rendering - setTimeout(f, 1); - }); - function signed_in(mesg) { setRememberMe(appBasePath); // Record which hub we're connected to. redux.getActions("account").setState({ hub: mesg.hub }); - console.log(`Signed into ${mesg.hub} at ${new Date()}`); + console.log(`Signed into conat server ${mesg.hub} at ${new Date()}`); if (first_login) { first_login = false; if (!should_load_target_url()) { diff --git a/src/packages/hub/README.md b/src/packages/hub/README.md index 6dd7878f6d..c9d1ffd336 100644 --- a/src/packages/hub/README.md +++ b/src/packages/hub/README.md @@ -1,12 +1,10 @@ -# The hub CoCalc web server. +# The hub CoCalc web server This code is part of https://github.com/sagemathinc/cocalc and isn't currently designed to be used standalone. Our plan is to refactor this code into smaller useful modules that are published under the @cocalc npm organization. This is a node.js process that serves _all_ of the following (possibly simultaneously): - static content - our mirror of the cdn and the results of webpack (packages/static) -- an http api as documented at https://doc.cocalc.com/api -- a websocket connection that client browsers use for sign in, account config, creating projects, etc. - a proxy server that connects client browsers to projects - project control server that creates, starts and stops projects running locally @@ -41,3 +39,26 @@ cd packages/hub; npx cocalc-hub-server ... --https-key=./selfsigned.key --https- ``` and the hub will use https instead of http. Simple as that. + +## Conat + +To run each component of conat separately. Do each of these in DIFFERENT terminals, with the env variables as possible examples in dev mode. + +Run the conat\-core socketio server on port 5000: + +```sh +~/cocalc/src/packages/hub$ PORT=5000 DEBUG=cocalc:* DEBUG_CONSOLE=yes pnpm cocalc-hub-server --conat-core +``` + +Run the conat\-api server on port 5002, but pointed out our port 5000 socketio server + +```sh +CONAT_SERVER=http://localhost:5000/3fa218e5-7196-4020-8b30-e2127847cc4f/port/5002 DEBUG=cocalc:*proxy* DEBUG_CONSOLE=yes pnpm cocalc-hub-server --conat-api +``` + +Run one or more persist servers: + +``` +CONAT_SERVER=http://localhost:5000/3fa218e5-7196-4020-8b30-e2127847cc4f/port/5002 DEBUG=cocalc:* DEBUG_CONSOLE=yes pnpm cocalc-hub-server --conat-persist +``` + diff --git a/src/packages/hub/api/handler.coffee b/src/packages/hub/api/handler.coffee deleted file mode 100644 index 63fc81a0d6..0000000000 --- a/src/packages/hub/api/handler.coffee +++ /dev/null @@ -1,172 +0,0 @@ -######################################################################### -# This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. -# License: MS-RSL – see LICENSE.md for details -######################################################################### - -### -API for handling the messages described packages/util/message.js - -MS-RSL, (c) 2017, SageMath, Inc. -### - -async = require('async') - -{getAccountWithApiKey} = require("@cocalc/server/api/manage"); - -Cache = require('lru-cache') -auth_cache = new Cache(max:100, ttl:60000) - -misc = require('@cocalc/util/misc') -{defaults, required} = misc - -messages = require('@cocalc/util/message') - -{ HELP_EMAIL } = require("@cocalc/util/theme") - -{Client} = require('../client') - -log = (name, logger) -> - return (m) -> logger.debug("API.#{name}: #{m}") - -exports.http_message_api_v1 = (opts) -> - try - opts = defaults opts, - event : required - body : required - api_key : required - database : required - projectControl : required - ip_address : required - logger : required - cb : required - catch err - opts.cb(err) - return - dbg = log('http_message_api_v1', opts.logger) - dbg("event=#{JSON.stringify(opts.event)}, body=#{JSON.stringify(opts.body)}") - - f = messages[opts.event] - if not f? - opts.cb("unknown endpoint '#{opts.event}'") - return - - if not messages.api_messages[opts.event] - opts.cb("endpoint '#{opts.event}' is not part of the HTTP API") - return - - try - mesg = f(opts.body, true) - catch err - opts.cb("invalid parameters '#{err}'") - return - - if mesg.event == 'query' and mesg.multi_response - otps.cb("multi_response queries aren't supported") - return - - # client often expects id to be defined. - mesg.id ?= misc.uuid() - - client = resp = undefined - async.series([ - (cb) -> - get_client - api_key : opts.api_key - logger : opts.logger - database : opts.database - projectControl : opts.projectControl - ip_address : opts.ip_address - cb : (err, c) -> - client = c; cb(err) - (cb) -> - handle_message - client : client - mesg : mesg - logger : opts.logger - cb : (err, r) -> - resp = r; cb(err) - ], (err) -> - if err - dbg("#{err} - #{JSON.stringify(resp)}") - opts.cb(err, resp) - ) - -get_client = (opts) -> - opts = defaults opts, - api_key : required - logger : required - database : required - projectControl : required - ip_address : required - cb : required - dbg = log('get_client', opts.logger) - dbg() - - account_id = auth_cache.get(opts.api_key) - - async.series([ - (cb) -> - if account_id - cb() - else - try - x = await getAccountWithApiKey(opts.api_key) - account_id = x.account_id ? x.project_id - if not account_id? - cb("No account found. Is your API key wrong?") - return - # briefly cache api key. see "expire" time in ms above. - auth_cache.set(opts.api_key, account_id) - cb() - catch err - cb(err) - return - (cb) -> - # check if user is banned: - opts.database.is_banned_user - account_id : account_id - cb : (err, is_banned) -> - if err - cb(err) - return - if is_banned - cb("User is BANNED. If this is a mistake, please contact #{HELP_EMAIL}") - return - cb() - - ], (err) -> - if err - opts.cb(err) - return - options = - logger : opts.logger - database : opts.database - projectControl : opts.projectControl - client = new Client(options) - client.push_to_client = (mesg, cb) => - client.emit('push_to_client', mesg) - cb?() - client.ip_address = opts.ip_address - client.account_id = account_id - opts.cb(undefined, client) - ) - -handle_message = (opts) -> - opts = defaults opts, - mesg : required - client : required - logger : undefined - cb : required - dbg = log('handle_message', opts.logger) - dbg("#{JSON.stringify(opts.mesg)}, #{opts.client.id}") - name = "mesg_#{opts.mesg.event}" - f = opts.client[name] - if not f? - opts.cb("unknown message event type '#{opts.mesg.event}'") - return - opts.client.once 'push_to_client', (mesg) -> - opts.cb(undefined, mesg) - f(opts.mesg) - - - diff --git a/src/packages/hub/client.coffee b/src/packages/hub/client.coffee deleted file mode 100644 index 60dad8098a..0000000000 --- a/src/packages/hub/client.coffee +++ /dev/null @@ -1,1559 +0,0 @@ -######################################################################### -# This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. -# License: MS-RSL – see LICENSE.md for details -######################################################################### - -### -Client = a client that is connected via a persistent connection to the hub -### - -{EventEmitter} = require('events') -uuid = require('uuid') -async = require('async') -Cookies = require('cookies') # https://github.com/jed/cookies -misc = require('@cocalc/util/misc') -{defaults, required, to_safe_str} = misc -message = require('@cocalc/util/message') -access = require('./access') -clients = require('./clients').getClients() -auth = require('./auth') -local_hub_connection = require('./local_hub_connection') -hub_projects = require('./projects') -{StripeClient} = require('@cocalc/server/stripe/client') -{send_email, send_invite_email} = require('./email') -purchase_license = require('@cocalc/server/licenses/purchase').default -db_schema = require('@cocalc/util/db-schema') -{ escapeHtml } = require("escape-html") -{CopyPath} = require('./copy-path') -{ REMEMBER_ME_COOKIE_NAME } = require("@cocalc/backend/auth/cookie-names"); -generateHash = require("@cocalc/server/auth/hash").default; -passwordHash = require("@cocalc/backend/auth/password-hash").default; -jupyter_execute = require('@cocalc/server/jupyter/execute').execute; -jupyter_kernels = require('@cocalc/server/jupyter/kernels').default; -create_project = require("@cocalc/server/projects/create").default; -collab = require('@cocalc/server/projects/collab'); -delete_passport = require('@cocalc/server/auth/sso/delete-passport').delete_passport; -setEmailAddress = require("@cocalc/server/accounts/set-email-address").default; - -{one_result} = require("@cocalc/database") - -path_join = require('path').join -base_path = require('@cocalc/backend/base-path').default - -underscore = require('underscore') - -{callback, delay} = require('awaiting') -{callback2} = require('@cocalc/util/async-utils') - -{record_user_tracking} = require('@cocalc/database/postgres/user-tracking') -{project_has_network_access} = require('@cocalc/database/postgres/project-queries') -{is_paying_customer} = require('@cocalc/database/postgres/account-queries') -{get_personal_user} = require('@cocalc/database/postgres/personal') - -{RESEND_INVITE_INTERVAL_DAYS} = require("@cocalc/util/consts/invites") - -removeLicenseFromProject = require('@cocalc/server/licenses/remove-from-project').default -addLicenseToProject = require('@cocalc/server/licenses/add-to-project').default - -DEBUG2 = !!process.env.SMC_DEBUG2 - -REQUIRE_ACCOUNT_TO_EXECUTE_CODE = false - -# Temporarily to handle old clients for a few days. -JSON_CHANNEL = '\u0000' - -# Anti DOS parameters: -# If a client sends a burst of messages, we space handling them out by this many milliseconds: -# (this even includes keystrokes when using the terminal) -MESG_QUEUE_INTERVAL_MS = 0 -# If a client sends a massive burst of messages, we discard all but the most recent this many of them: -# The client *should* be implemented in a way so that this never happens, and when that is -# the case -- according to our loging -- we might switch to immediately banning clients that -# hit these limits... -MESG_QUEUE_MAX_COUNT = 300 -MESG_QUEUE_MAX_WARN = 50 - -# Any messages larger than this is dropped (it could take a long time to handle, by a de-JSON'ing attack, etc.). -# On the other hand, it is good to make this large enough that projects can save -MESG_QUEUE_MAX_SIZE_MB = 20 - -# How long to cache a positive authentication for using a project. -CACHE_PROJECT_AUTH_MS = 1000*60*15 # 15 minutes - -# How long all info about a websocket Client connection -# is kept in memory after a user disconnects. This makes it -# so that if they quickly reconnect, the connections to projects -# and other state doesn't have to be recomputed. -CLIENT_DESTROY_TIMER_S = 60*10 # 10 minutes -#CLIENT_DESTROY_TIMER_S = 0.1 # instant -- for debugging - -CLIENT_MIN_ACTIVE_S = 45 - - -class exports.Client extends EventEmitter - constructor: (opts) -> - super() - @_opts = defaults opts, - conn : undefined - logger : undefined - database : required - projectControl : required - host : undefined - port : undefined - personal : undefined - - @conn = @_opts.conn - @logger = @_opts.logger - @database = @_opts.database - @projectControl = @_opts.projectControl - - @_when_connected = new Date() - - @_messages = - being_handled : {} - total_time : 0 - count : 0 - - # The variable account_id is either undefined or set to the - # account id of the user that this session has successfully - # authenticated as. Use @account_id to decide whether or not - # it is safe to carry out a given action. - @account_id = undefined - - if @conn? - # has a persistent connection, e.g., NOT just used for an API - @init_conn() - else - @id = misc.uuid() - - @copy_path = new CopyPath(@) - - init_conn: => - # initialize everything related to persistent connections - @_data_handlers = {} - @_data_handlers[JSON_CHANNEL] = @handle_json_message_from_client - - # The persistent sessions that this client starts. - @compute_session_uuids = [] - - @install_conn_handlers() - - @ip_address = @conn.address.ip - - # A unique id -- can come in handy - @id = @conn.id - - # Setup remember-me related cookie handling - @cookies = {} - c = new Cookies(@conn.request) - @_remember_me_value = c.get(REMEMBER_ME_COOKIE_NAME) - - @check_for_remember_me() - - # Security measure: check every 5 minutes that remember_me - # cookie used for login is still valid. If the cookie is gone - # and this fails, user gets a message, and see that they must sign in. - @_remember_me_interval = setInterval(@check_for_remember_me, 1000*60*5) - - touch: (opts={}) => - if not @account_id # not logged in - opts.cb?('not logged in') - return - opts = defaults opts, - project_id : undefined - path : undefined - action : 'edit' - force : false - cb : undefined - # touch -- indicate by changing field in database that this user is active. - # We do this at most once every CLIENT_MIN_ACTIVE_S seconds, for given choice - # of project_id, path (unless force is true). - if not @_touch_lock? - @_touch_lock = {} - key = "#{opts.project_id}-#{opts.path}-#{opts.action}" - if not opts.force and @_touch_lock[key] - opts.cb?("touch lock") - return - opts.account_id = @account_id - @_touch_lock[key] = true - delete opts.force - @database.touch(opts) - - setTimeout((()=>delete @_touch_lock[key]), CLIENT_MIN_ACTIVE_S*1000) - - install_conn_handlers: () => - dbg = @dbg('install_conn_handlers') - if @_destroy_timer? - clearTimeout(@_destroy_timer) - delete @_destroy_timer - - @conn.on "data", (data) => - @handle_data_from_client(data) - - @conn.on "end", () => - dbg("connection: hub <--> client(id=#{@id}, address=#{@ip_address}) -- CLOSED") - @destroy() - ### - # I don't think this destroy_timer is of any real value at all unless - # we were to fully maintain client state while they are gone. Doing this - # is a serious liability, e.g., in a load-spike situation. - # CRITICAL -- of course we need to cancel all changefeeds when user disconnects, - # even temporarily, since messages could be dropped otherwise. (The alternative is to - # cache all messages in the hub, which has serious memory implications.) - @query_cancel_all_changefeeds() - # Actually destroy Client in a few minutes, unless user reconnects - # to this session. Often the user may have a temporary network drop, - # and we keep everything waiting for them for short time - # in case this happens. - @_destroy_timer = setTimeout(@destroy, 1000*CLIENT_DESTROY_TIMER_S) - ### - - dbg("connection: hub <--> client(id=#{@id}, address=#{@ip_address}) ESTABLISHED") - - dbg: (desc) => - if @logger?.debug - return (args...) => @logger.debug("Client(#{@id}).#{desc}:", args...) - else - return -> - - destroy: () => - dbg = @dbg('destroy') - dbg("destroy connection: hub <--> client(id=#{@id}, address=#{@ip_address}) -- CLOSED") - - if @id - # cancel any outstanding queries. - @database.cancel_user_queries(client_id:@id) - - delete @_project_cache - clearInterval(@_remember_me_interval) - @closed = true - @emit('close') - @compute_session_uuids = [] - c = clients[@id] - delete clients[@id] - dbg("num_clients=#{misc.len(clients)}") - if c? and c.call_callbacks? - for id,f of c.call_callbacks - f("connection closed") - delete c.call_callbacks - for h in local_hub_connection.all_local_hubs() - h.free_resources_for_client_id(@id) - - remember_me_failed: (reason) => - return if not @conn? - @signed_out() # so can't do anything with projects, etc. - @push_to_client(message.remember_me_failed(reason:reason)) - - get_personal_user: () => - if @account_id or not @conn? or not @_opts.personal - # there is only one account - return - dbg = @dbg("check_for_remember_me") - dbg("personal mode") - try - signed_in_mesg = {account_id:await get_personal_user(@database), event:'signed_in'} - # sign them in if not already signed in (due to async this could happen - # by get_personal user getting called twice at once). - if @account_id != signed_in_mesg.account_id - signed_in_mesg.hub = @_opts.host + ':' + @_opts.port - @signed_in(signed_in_mesg) - @push_to_client(signed_in_mesg) - catch err - dbg("remember_me: personal mode error", err.toString()) - @remember_me_failed("error getting personal user -- #{err}") - return - - check_for_remember_me: () => - return if not @conn? - dbg = @dbg("check_for_remember_me") - - if @_opts.personal - @get_personal_user() - return - - value = @_remember_me_value - if not value? - @remember_me_failed("no remember_me cookie") - return - x = value.split('$') - if x.length != 4 - @remember_me_failed("invalid remember_me cookie") - return - try - hash = generateHash(x[0], x[1], x[2], x[3]) - catch err - dbg("unable to generate hash from '#{value}' -- #{err}") - @remember_me_failed("invalid remember_me cookie") - return - - dbg("checking for remember_me cookie with hash='#{hash.slice(0,15)}...'") # don't put all in log -- could be dangerous - @database.get_remember_me - hash : hash - cb : (error, signed_in_mesg) => - dbg("remember_me: got ", error) - if error - @remember_me_failed("error accessing database") - return - if not signed_in_mesg or not signed_in_mesg.account_id - @remember_me_failed("remember_me deleted or expired") - return - # sign them in if not already signed in - if @account_id != signed_in_mesg.account_id - # DB only tells us the account_id, but the hub might have changed from last time - signed_in_mesg.hub = @_opts.host + ':' + @_opts.port - @hash_session_id = hash - @signed_in(signed_in_mesg) - @push_to_client(signed_in_mesg) - - push_to_client: (mesg, cb) => - ### - Pushing messages to this particular connected client - ### - if @closed - cb?("disconnected") - return - dbg = @dbg("push_to_client") - - if mesg.event != 'pong' - dbg("hub --> client (client=#{@id}): #{misc.trunc(to_safe_str(mesg),300)}") - #dbg("hub --> client (client=#{@id}): #{misc.trunc(JSON.stringify(mesg),1000)}") - #dbg("hub --> client (client=#{@id}): #{JSON.stringify(mesg)}") - - if mesg.id? - start = @_messages.being_handled[mesg.id] - if start? - time_taken = new Date() - start - delete @_messages.being_handled[mesg.id] - @_messages.total_time += time_taken - @_messages.count += 1 - avg = Math.round(@_messages.total_time / @_messages.count) - dbg("[#{time_taken} mesg_time_ms] [#{avg} mesg_avg_ms] -- mesg.id=#{mesg.id}") - - # If cb *is* given and mesg.id is *not* defined, then - # we also setup a listener for a response from the client. - listen = cb? and not mesg.id? - if listen - # This message is not a response to a client request. - # Instead, we are initiating a request to the user and we - # want a result back (hence cb? being defined). - mesg.id = misc.uuid() - if not @call_callbacks? - @call_callbacks = {} - @call_callbacks[mesg.id] = cb - f = () => - g = @call_callbacks?[mesg.id] - if g? - delete @call_callbacks[mesg.id] - g("timed out") - setTimeout(f, 15000) # timeout after some seconds - - t = new Date() - data = misc.to_json_socket(mesg) - tm = new Date() - t - if tm > 10 - dbg("mesg.id=#{mesg.id}: time to json=#{tm}ms; length=#{data.length}; value='#{misc.trunc(data, 500)}'") - @push_data_to_client(data) - if not listen - cb?() - return - - push_data_to_client: (data) -> - return if not @conn? - if @closed - return - @conn.write(data) - - error_to_client: (opts) => - opts = defaults opts, - id : undefined - error : required - if opts.error instanceof Error - # Javascript Errors as come up with exceptions don't JSON. - # Since the point is just to show an error to the client, - # it is better to send back the string! - opts.error = opts.error.toString() - @push_to_client(message.error(id:opts.id, error:opts.error)) - - success_to_client: (opts) => - opts = defaults opts, - id : required - @push_to_client(message.success(id:opts.id)) - - signed_in: (signed_in_mesg) => - return if not @conn? - # Call this method when the user has successfully signed in. - - @signed_in_mesg = signed_in_mesg # save it, since the properties are handy to have. - - # Record that this connection is authenticated as user with given uuid. - @account_id = signed_in_mesg.account_id - - # Get user's group from database. - @get_groups() - - signed_out: () => - @account_id = undefined - - # Setting and getting HTTP-only cookies via Primus + AJAX - get_cookie: (opts) => - opts = defaults opts, - name : required - cb : required # cb(undefined, value) - if not @conn?.id? - # no connection or connection died - return - @once("get_cookie-#{opts.name}", (value) -> opts.cb(value)) - @push_to_client(message.cookies(id:@conn.id, get:opts.name, url:path_join(base_path, "cookies"))) - - - invalidate_remember_me: (opts) => - return if not @conn? - - opts = defaults opts, - cb : required - - if @hash_session_id? - @database.delete_remember_me - hash : @hash_session_id - cb : opts.cb - else - opts.cb() - - ### - Our realtime socket connection might only support one connection - between the client and - server, so we multiplex multiple channels over the same - connection. There is one base channel for JSON messages called - JSON_CHANNEL, which themselves can be routed to different - callbacks, etc., by the client code. There are 16^4-1 other - channels, which are for sending raw data. The raw data messages - are prepended with a UTF-16 character that identifies the - channel. The channel character is random (which might be more - secure), and there is no relation between the channels for two - distinct clients. - ### - - handle_data_from_client: (data) => - return if not @conn? - dbg = @dbg("handle_data_from_client") - ## Only enable this when doing low level debugging -- performance impacts AND leakage of dangerous info! - if DEBUG2 - dbg("handle_data_from_client('#{misc.trunc(data.toString(),400)}')") - - # TODO: THIS IS A SIMPLE anti-DOS measure; it might be too - # extreme... we shall see. It prevents a number of attacks, - # e.g., users storing a multi-gigabyte worksheet title, - # etc..., which would (and will) otherwise require care with - # every single thing we store. - - # TODO: the two size things below should be specific messages (not generic error_to_client), and - # be sensibly handled by the client. - if data.length >= MESG_QUEUE_MAX_SIZE_MB * 10000000 - # We don't parse it, we don't look at it, we don't know it's id. This shouldn't ever happen -- and probably would only - # happen because of a malicious attacker. JSON parsing arbitrarily large strings would - # be very dangerous, and make crashing the server way too easy. - # We just respond with this error below. The client should display to the user all id-less errors. - msg = "The server ignored a huge message since it exceeded the allowed size limit of #{MESG_QUEUE_MAX_SIZE_MB}MB. Please report what caused this if you can." - @logger?.error(msg) - @error_to_client(error:msg) - return - - if not @_handle_data_queue? - @_handle_data_queue = [] - - # The rest of the function is basically the same as "h(data.slice(1))", except that - # it ensure that if there is a burst of messages, then (1) we handle at most 1 message - # per client every MESG_QUEUE_INTERVAL_MS, and we drop messages if there are too many. - # This is an anti-DOS measure. - @_handle_data_queue.push([@handle_json_message_from_client, data]) - - if @_handle_data_queue_empty_function? - return - - # define a function to empty the queue - @_handle_data_queue_empty_function = () => - if @_handle_data_queue.length == 0 - # done doing all tasks - delete @_handle_data_queue_empty_function - return - - if @_handle_data_queue.length > MESG_QUEUE_MAX_WARN - dbg("MESG_QUEUE_MAX_WARN(=#{MESG_QUEUE_MAX_WARN}) exceeded (=#{@_handle_data_queue.length}) -- just a warning") - - # drop oldest message to keep - if @_handle_data_queue.length > MESG_QUEUE_MAX_COUNT - dbg("MESG_QUEUE_MAX_COUNT(=#{MESG_QUEUE_MAX_COUNT}) exceeded (=#{@_handle_data_queue.length}) -- drop oldest messages") - while @_handle_data_queue.length > MESG_QUEUE_MAX_COUNT - discarded_mesg = @_handle_data_queue.shift() - data = discarded_mesg?[1] - dbg("discarded_mesg='#{misc.trunc(data?.toString?(),1000)}'") - - - # get task - task = @_handle_data_queue.shift() - # do task - task[0](task[1]) - # do next one in >= MESG_QUEUE_INTERVAL_MS - setTimeout( @_handle_data_queue_empty_function, MESG_QUEUE_INTERVAL_MS ) - - @_handle_data_queue_empty_function() - - register_data_handler: (h) -> - return if not @conn? - # generate a channel character that isn't already taken -- if these get too large, - # this will break (see, e.g., http://blog.fgribreau.com/2012/05/how-to-fix-could-not-decode-text-frame.html); - # however, this is a counter for *each* individual user connection, so they won't get too big. - # Ultimately, we'll redo things to use primus/websocket channel support, which should be much more powerful - # and faster. - if not @_last_channel? - @_last_channel = 1 - while true - @_last_channel += 1 - channel = String.fromCharCode(@_last_channel) - if not @_data_handlers[channel]? - break - @_data_handlers[channel] = h - return channel - - ### - Message handling functions: - - Each function below that starts with mesg_ handles a given - message type (an event). The implementations of many of the - handlers are somewhat long/involved, so the function below - immediately calls another function defined elsewhere. This will - make it easier to refactor code to other modules, etc., later. - This approach also clarifies what exactly about this object - is used to implement the relevant functionality. - ### - handle_json_message_from_client: (data) => - return if not @conn? - if @_ignore_client - return - try - mesg = misc.from_json_socket(data) - catch error - @logger?.error("error parsing incoming mesg (invalid JSON): #{mesg}") - return - dbg = @dbg('handle_json_message_from_client') - if mesg.event != 'ping' - dbg("hub <-- client: #{misc.trunc(to_safe_str(mesg), 120)}") - - # check for message that is coming back in response to a request from the hub - if @call_callbacks? and mesg.id? - f = @call_callbacks[mesg.id] - if f? - delete @call_callbacks[mesg.id] - f(undefined, mesg) - return - - if mesg.id? - @_messages.being_handled[mesg.id] = new Date() - - handler = @["mesg_#{mesg.event}"] - if handler? - try - await handler(mesg) - catch err - # handler *should* handle any possible error, but just in case something - # not expected goes wrong... we do this - @error_to_client(id:mesg.id, error:"${err}") - else - @push_to_client(message.error(error:"Hub does not know how to handle a '#{mesg.event}' event.", id:mesg.id)) - if mesg.event == 'get_all_activity' - dbg("ignoring all further messages from old client=#{@id}") - @_ignore_client = true - - mesg_sign_out: (mesg) => - if not @account_id? - @push_to_client(message.error(id:mesg.id, error:"not signed in")) - return - - if mesg.everywhere - # invalidate all remember_me cookies - @database.invalidate_all_remember_me - account_id : @account_id - @signed_out() # deletes @account_id... so must be below database call above - # invalidate the remember_me on this browser - @invalidate_remember_me - cb:(error) => - @dbg('mesg_sign_out')("signing out: #{mesg.id}, #{error}") - if error - @push_to_client(message.error(id:mesg.id, error:error)) - else - @push_to_client(message.signed_out(id:mesg.id)) - - # Messages: Password/email address management - mesg_change_email_address: (mesg) => - try - await setEmailAddress - account_id: @account_id - email_address: mesg.new_email_address - password: mesg.password - @push_to_client(message.changed_email_address(id:mesg.id)) - catch err - @error_to_client(id:mesg.id, error:err) - - mesg_send_verification_email: (mesg) => - auth = require('./auth') - auth.verify_email_send_token - account_id : mesg.account_id - only_verify : mesg.only_verify ? true - database : @database - cb : (err) => - if err - @error_to_client(id:mesg.id, error:err) - else - @success_to_client(id:mesg.id) - - mesg_unlink_passport: (mesg) => - if not @account_id? - @error_to_client(id:mesg.id, error:"must be logged in") - else - opts = - account_id : @account_id - strategy : mesg.strategy - id : mesg.id - cb : (err) => - if err - @error_to_client(id:mesg.id, error:err) - else - @success_to_client(id:mesg.id) - delete_passport(@database, opts) - - # Messages: Account settings - get_groups: (cb) => - # see note above about our "infinite caching". Maybe a bad idea. - if @groups? - cb?(undefined, @groups) - return - @database.get_account - columns : ['groups'] - account_id : @account_id - cb : (err, r) => - if err - cb?(err) - else - @groups = r['groups'] - cb?(undefined, @groups) - - # Messages: Log errors that client sees so we can also look at them - mesg_log_client_error: (mesg) => - @dbg('mesg_log_client_error')(mesg.error) - if not mesg.type? - mesg.type = "error" - if not mesg.error? - mesg.error = "error" - @database.log_client_error - event : mesg.type - error : mesg.error - account_id : @account_id - cb : (err) => - if not mesg.id? - return - if err - @error_to_client(id:mesg.id, error:err) - else - @success_to_client(id:mesg.id) - - mesg_webapp_error: (mesg) => - # Tell client we got it, thanks: - @success_to_client(id:mesg.id) - # Now do something with it. - @dbg('mesg_webapp_error')(mesg.msg) - mesg = misc.copy_without(mesg, 'event') - mesg.account_id = @account_id - @database.webapp_error(mesg) - - # Messages: Project Management - get_project: (mesg, permission, cb) => - ### - How to use this: Either call the callback with the project, or if an error err - occured, call @error_to_client(id:mesg.id, error:err) and *NEVER* - call the callback. This function is meant to be used in a bunch - of the functions below for handling requests. - - mesg -- must have project_id field - permission -- must be "read" or "write" - cb(err, project) - *NOTE*: on failure, if mesg.id is defined, then client will receive - an error message; the function calling get_project does *NOT* - have to send the error message back to the client! - ### - dbg = @dbg('get_project') - - err = undefined - if not mesg.project_id? - err = "mesg must have project_id attribute -- #{to_safe_str(mesg)}" - else if not @account_id? - err = "user must be signed in before accessing projects" - - if err - if mesg.id? - @error_to_client(id:mesg.id, error:err) - cb(err) - return - - key = mesg.project_id + permission - project = @_project_cache?[key] - if project? - # Use the cached project so we don't have to re-verify authentication - # for the user again below, which - # is very expensive. This cache does expire, in case user - # is kicked out of the project. - cb(undefined, project) - return - - dbg() - async.series([ - (cb) => - switch permission - when 'read' - access.user_has_read_access_to_project - project_id : mesg.project_id - account_id : @account_id - account_groups : @groups - database : @database - cb : (err, result) => - if err - cb("Read access denied -- #{err}") - else if not result - cb("User #{@account_id} does not have read access to project #{mesg.project_id}") - else - # good to go - cb() - when 'write' - access.user_has_write_access_to_project - database : @database - project_id : mesg.project_id - account_groups : @groups - account_id : @account_id - cb : (err, result) => - if err - cb("Write access denied -- #{err}") - else if not result - cb("User #{@account_id} does not have write access to project #{mesg.project_id}") - else - # good to go - cb() - else - cb("Internal error -- unknown permission type '#{permission}'") - ], (err) => - if err - if mesg.id? - @error_to_client(id:mesg.id, error:err) - dbg("error -- #{err}") - cb(err) - else - project = hub_projects.new_project(mesg.project_id, @database, @projectControl) - @database.touch_project(project_id:mesg.project_id) - @_project_cache ?= {} - @_project_cache[key] = project - # cache for a while - setTimeout((()=>delete @_project_cache?[key]), CACHE_PROJECT_AUTH_MS) - dbg("got project; caching and returning") - cb(undefined, project) - ) - - - mesg_project_exec: (mesg) => - if mesg.command == "ipython-notebook" - # we just drop these messages, which are from old non-updated clients (since we haven't - # written code yet to not allow them to connect -- TODO!). - return - @get_project mesg, 'write', (err, project) => - if err - return - project.call - mesg : mesg - timeout : mesg.timeout - cb : (err, resp) => - if err - @error_to_client(id:mesg.id, error:err) - else - @push_to_client(resp) - - mesg_copy_path_between_projects: (mesg) => - @copy_path.copy(mesg) - - mesg_copy_path_status: (mesg) => - @copy_path.status(mesg) - - mesg_copy_path_delete: (mesg) => - @copy_path.delete(mesg) - - mesg_local_hub: (mesg) => - ### - Directly communicate with the local hub. If the - client has write access to the local hub, there's no - reason they shouldn't be allowed to send arbitrary - messages directly (they could anyways from the terminal). - ### - dbg = @dbg('mesg_local_hub') - dbg("hub --> local_hub: ", mesg) - @get_project mesg, 'write', (err, project) => - if err - return - if not mesg.message? - # in case the message itself is invalid -- is possible - @error_to_client(id:mesg.id, error:"message must be defined") - return - - if mesg.message.event == 'project_exec' and mesg.message.command == "ipython-notebook" - # we just drop these messages, which are from old non-updated clients (since we haven't - # written code yet to not allow them to connect -- TODO!). - return - - # It's extremely useful if the local hub has a way to distinguish between different clients who are - # being proxied through the same hub. - mesg.message.client_id = @id - - # Make the actual call - project.call - mesg : mesg.message - timeout : mesg.timeout - multi_response : mesg.multi_response - cb : (err, resp) => - if err - dbg("ERROR: #{err} calling message #{misc.to_json(mesg.message)}") - @error_to_client(id:mesg.id, error:err) - else - if not mesg.multi_response - resp.id = mesg.id - @push_to_client(resp) - - # this is an async function - allow_urls_in_emails: (project_id) => - is_paying = await is_paying_customer(@database, @account_id) - has_network = await project_has_network_access(@database, project_id) - return is_paying or has_network - - mesg_invite_collaborator: (mesg) => - @touch() - dbg = @dbg('mesg_invite_collaborator') - #dbg("mesg: #{misc.to_json(mesg)}") - @get_project mesg, 'write', (err, project) => - if err - return - locals = - email_address : undefined - done : false - settings : undefined - - # SECURITY NOTE: mesg.project_id is valid and the client has write access, since otherwise, - # the @get_project function above wouldn't have returned without err... - async.series([ - (cb) => - @database.add_user_to_project - project_id : mesg.project_id - account_id : mesg.account_id - group : 'collaborator' # in future will be "invite_collaborator", once implemented - cb : cb - - (cb) => - # only send an email when there is an mesg.email body to send. - # we want to make it explicit when they're sent, and implicitly disable it for API usage. - if not mesg.email? - locals.done = true - cb() - - (cb) => - if locals.done - cb(); return - - @database._query - query : "SELECT email_address FROM accounts" - where : "account_id = $::UUID" : mesg.account_id - cb : one_result 'email_address', (err, x) => - locals.email_address = x - cb(err) - - (cb) => - if (not locals.email_address) or locals.done - cb(); return - - # INFO: for testing this, you have to reset the invite field each time you sent yourself an invitation - # in psql: UPDATE projects SET invite = NULL WHERE project_id = ''; - @database.when_sent_project_invite - project_id : mesg.project_id - to : locals.email_address - cb : (err, when_sent) => - #console.log("mesg_invite_collaborator email #{locals.email_address}, #{err}, #{when_sent}") - if err - cb(err) - else if when_sent >= misc.days_ago(7) # successfully sent < one week ago -- don't again - locals.done = true - cb() - else - cb() - - (cb) => - if locals.done or (not locals.email_address) - cb() - return - @database.get_server_settings_cached - cb : (err, settings) => - if err - cb(err) - else if not settings? - cb("no server settings -- no database connection?") - else - locals.settings = settings - cb() - - (cb) => - if locals.done or (not locals.email_address) - dbg("NOT send_email invite to #{locals.email_address}") - cb() - return - - ## do not send email if project doesn't have network access (and user is not a paying customer) - #if (not await is_paying_customer(@database, @account_id) and not await project_has_network_access(@database, mesg.project_id)) - # dbg("NOT send_email invite to #{locals.email_address} -- due to project lacking network access (and user not a customer)") - # return - - # we always send invite emails. for non-upgraded projects, we sanitize the content of the body - # ATTN: this must harmonize with @cocalc/frontend/projects → allow_urls_in_emails - # also see mesg_invite_noncloud_collaborators - - dbg("send_email invite to #{locals.email_address}") - # available message fields - # mesg.title - title of project - # mesg.link2proj - # mesg.replyto - # mesg.replyto_name - # mesg.email - body of email - # mesg.subject - - # send an email to the user -- async, not blocking user. - # TODO: this can take a while -- we need to take some action - # if it fails, e.g., change a setting in the projects table! - if mesg.replyto_name? - subject = "#{mesg.replyto_name} invited you to collaborate on CoCalc in project '#{mesg.title}'" - else - subject = "Invitation to CoCalc for collaborating in project '#{mesg.title}'" - # override subject if explicitly given - if mesg.subject? - subject = mesg.subject - - send_invite_email - to : locals.email_address - subject : subject - email : mesg.email - email_address : locals.email_address - title : mesg.title - allow_urls : await @allow_urls_in_emails(mesg.project_id) - replyto : mesg.replyto ? locals.settings.organization_email - replyto_name : mesg.replyto_name - link2proj : mesg.link2proj - settings : locals.settings - cb : (err) => - if err - dbg("FAILED to send email to #{locals.email_address} -- err=#{misc.to_json(err)}") - @database.sent_project_invite - project_id : mesg.project_id - to : locals.email_address - error : err - cb(err) # call the cb one scope up so that the client is informed that we sent the invite (or not) - - ], (err) => - if err - @error_to_client(id:mesg.id, error:err) - else - @push_to_client(message.success(id:mesg.id)) - ) - - mesg_add_license_to_project: (mesg) => - dbg = @dbg('mesg_add_license_to_project') - dbg() - @touch() - @_check_project_access mesg.project_id, (err) => - if err - dbg("failed -- #{err}") - @error_to_client(id:mesg.id, error:"must have write access to #{mesg.project_id} -- #{err}") - return - try - await addLicenseToProject({project_id:mesg.project_id, license_id:mesg.license_id}) - @success_to_client(id:mesg.id) - catch err - @error_to_client(id:mesg.id, error:"#{err}") - - mesg_remove_license_from_project: (mesg) => - dbg = @dbg('mesg_remove_license_from_project') - dbg() - @touch() - @_check_project_access mesg.project_id, (err) => - if err - dbg("failed -- #{err}") - @error_to_client(id:mesg.id, error:"must have write access to #{mesg.project_id} -- #{err}") - return - try - await removeLicenseFromProject({project_id:mesg.project_id, license_id:mesg.license_id}) - @success_to_client(id:mesg.id) - catch err - @error_to_client(id:mesg.id, error:"#{err}") - - mesg_invite_noncloud_collaborators: (mesg) => - dbg = @dbg('mesg_invite_noncloud_collaborators') - - # - # Uncomment this in case of attack by evil forces: - # - ## @error_to_client(id:mesg.id, error:"inviting collaborators who do not already have a cocalc account to projects is currently disabled due to abuse"); - ## return - - # Otherwise we always allow sending email invites - # The body is sanitized and not allowed to contain any URLs (anti-spam), unless - # (a) the sender is a paying customer or (b) the project has network access. - # - # ATTN: this must harmonize with @cocalc/frontend/projects → allow_urls_in_emails - # also see mesg_invite_collaborator - - @touch() - @get_project mesg, 'write', (err, project) => - if err - return - - if mesg.to.length > 1024 - @error_to_client(id:mesg.id, error:"Specify less recipients when adding collaborators to project.") - return - - # users to invite - to = (x for x in mesg.to.replace(/\s/g,",").replace(/;/g,",").split(',') when x) - - invite_user = (email_address, cb) => - dbg("inviting #{email_address}") - if not misc.is_valid_email_address(email_address) - cb("invalid email address '#{email_address}'") - return - email_address = misc.lower_email_address(email_address) - if email_address.length >= 128 - # if an attacker tries to embed a spam in the email address itself (e.g, wstein+spam_message@gmail.com), then - # at least we can limit its size. - cb("email address must be at most 128 characters: '#{email_address}'") - return - - locals = - done : false - account_id : undefined - settings : undefined - - async.series([ - # already have an account? - (cb) => - @database.account_exists - email_address : email_address - cb : (err, _account_id) => - dbg("account_exists: #{err}, #{_account_id}") - locals.account_id = _account_id - cb(err) - (cb) => - if locals.account_id - dbg("user #{email_address} already has an account -- add directly") - # user has an account already - locals.done = true - @database.add_user_to_project - project_id : mesg.project_id - account_id : locals.account_id - group : 'collaborator' - cb : cb - else - dbg("user #{email_address} doesn't have an account yet -- may send email (if we haven't recently)") - # create trigger so that when user eventually makes an account, - # they will be added to the project. - @database.account_creation_actions - email_address : email_address - action : {action:'add_to_project', group:'collaborator', project_id:mesg.project_id} - ttl : 60*60*24*14 # valid for 14 days - cb : cb - (cb) => - if locals.done - cb() - else - @database.when_sent_project_invite - project_id : mesg.project_id - to : email_address - cb : (err, when_sent) => - if err - cb(err) - else if when_sent >= misc.days_ago(RESEND_INVITE_INTERVAL_DAYS) # successfully sent this long ago -- don't again - locals.done = true - cb() - else - cb() - - (cb) => - if locals.done - cb() - return - @database.get_server_settings_cached - cb: (err, settings) => - if err - cb(err) - else if not settings? - cb("no server settings -- no database connection?") - else - locals.settings = settings - cb() - - (cb) => - if locals.done - dbg("NOT send_email invite to #{email_address}") - cb() - return - - # send an email to the user -- async, not blocking user. - # TODO: this can take a while -- we need to take some action - # if it fails, e.g., change a setting in the projects table! - subject = "CoCalc Invitation" - # override subject if explicitly given - if mesg.subject? - subject = mesg.subject - - dbg("send_email invite to #{email_address}") - send_invite_email - to : email_address - subject : subject - email : mesg.email - email_address : email_address - title : mesg.title - allow_urls : await @allow_urls_in_emails(mesg.project_id) - replyto : mesg.replyto ? locals.settings.organization_email - replyto_name : mesg.replyto_name - link2proj : mesg.link2proj - settings : locals.settings - cb : (err) => - if err - dbg("FAILED to send email to #{email_address} -- err=#{misc.to_json(err)}") - @database.sent_project_invite - project_id : mesg.project_id - to : email_address - error : err - cb(err) # call the cb one scope up so that the client is informed that we sent the invite (or not) - - ], cb) - - async.map to, invite_user, (err, results) => - if err - @error_to_client(id:mesg.id, error:err) - else - @push_to_client(message.invite_noncloud_collaborators_resp(id:mesg.id, mesg:"Invited #{mesg.to} to collaborate on a project.")) - - mesg_remove_collaborator: (mesg) => - @touch() - @get_project mesg, 'write', (err, project) => - if err - return - # See "Security note" in mesg_invite_collaborator - @database.remove_collaborator_from_project - project_id : mesg.project_id - account_id : mesg.account_id - cb : (err) => - if err - @error_to_client(id:mesg.id, error:err) - else - @push_to_client(message.success(id:mesg.id)) - - # NOTE: this is different than invite_collab, in that it is - # much more similar to remove_collaborator. It also supports - # adding multiple collabs to multiple projects (NOT in one - # transaction, though). - mesg_add_collaborator: (mesg) => - @touch() - projects = mesg.project_id - accounts = mesg.account_id - tokens = mesg.token_id - - is_single_token = false - if tokens - if not misc.is_array(tokens) - is_single_token = true - tokens = [tokens] - projects = ('' for _ in [0...tokens.length]) # will get mutated below as tokens are used - if not misc.is_array(projects) - projects = [projects] - if not misc.is_array(accounts) - accounts = [accounts] - - try - await collab.add_collaborators_to_projects(@database, @account_id, accounts, projects, tokens) - resp = message.success(id:mesg.id) - if tokens - # Tokens determine the projects, and it maybe useful to the client to know what - # project they just got added to! - if is_single_token - resp.project_id = projects[0] - else - resp.project_id = projects - @push_to_client(resp) - catch err - @error_to_client(id:mesg.id, error:"#{err}") - - mesg_remove_blob_ttls: (mesg) => - if not @account_id? - @push_to_client(message.error(id:mesg.id, error:"not yet signed in")) - else - @database.remove_blob_ttls - uuids : mesg.uuids - cb : (err) => - if err - @error_to_client(id:mesg.id, error:err) - else - @push_to_client(message.success(id:mesg.id)) - - mesg_version: (mesg) => - # The version of the client... - @smc_version = mesg.version - @dbg('mesg_version')("client.smc_version=#{mesg.version}") - {version} = await require('./servers/server-settings').default() - if mesg.version < version.version_recommended_browser ? 0 - @push_version_update() - - push_version_update: => - {version} = await require('./servers/server-settings').default() - @push_to_client(message.version(version:version.version_recommended_browser, min_version:version.version_min_browser)) - if version.version_min_browser and @smc_version < version.version_min_browser - # Client is running an unsupported bad old version. - # Brutally disconnect client! It's critical that they upgrade, since they are - # causing problems or have major buggy code. - if new Date() - @_when_connected <= 30000 - # If they just connected, kill the connection instantly - @conn.end() - else - # Wait 1 minute to give them a chance to save data... - setTimeout((()=>@conn.end()), 60000) - - _user_is_in_group: (group) => - return @groups? and group in @groups - - assert_user_is_in_group: (group, cb) => - @get_groups (err) => - if not err and not @_user_is_in_group('admin') # user_is_in_group works after get_groups is called. - err = "must be logged in and a member of the admin group" - cb(err) - - mesg_project_set_quotas: (mesg) => - if not misc.is_valid_uuid_string(mesg.project_id) - @error_to_client(id:mesg.id, error:"invalid project_id") - return - project = undefined - dbg = @dbg("mesg_project_set_quotas(project_id='#{mesg.project_id}')") - async.series([ - (cb) => - @assert_user_is_in_group('admin', cb) - (cb) => - dbg("update base quotas in the database") - @database.set_project_settings - project_id : mesg.project_id - settings : misc.copy_without(mesg, ['event', 'id']) - cb : cb - (cb) => - dbg("get project from compute server") - try - project = await @projectControl(mesg.project_id) - cb() - catch err - cb(err) - (cb) => - dbg("determine total quotas and apply") - try - project.setAllQuotas() - cb() - catch err - cb(err) - ], (err) => - if err - @error_to_client(id:mesg.id, error:"problem setting project quota -- #{err}") - else - @push_to_client(message.success(id:mesg.id)) - ) - - ### - Public/published projects data - ### - path_is_in_public_paths: (path, paths) => - return misc.path_is_in_public_paths(path, misc.keys(paths)) - - get_public_project: (opts) => - ### - Get a compute.Project object, or cb an error if the given - path in the project isn't public. This is just like getting - a project, but first ensures that given path is public. - ### - opts = defaults opts, - project_id : undefined - path : undefined - use_cache : true - cb : required - - if not opts.project_id? - opts.cb("get_public_project: project_id must be defined") - return - - if not opts.path? - opts.cb("get_public_project: path must be defined") - return - - # determine if path is public in given project, without using cache to determine paths; this *does* cache the result. - @database.path_is_public - project_id : opts.project_id - path : opts.path - cb : (err, is_public) => - if err - opts.cb(err) - return - if is_public - try - opts.cb(undefined, await @projectControl(opts.project_id)) - catch err - opts.cb(err) - else - # no - opts.cb("path '#{opts.path}' of project with id '#{opts.project_id}' is not public") - - mesg_copy_public_path_between_projects: (mesg) => - @touch() - if not mesg.src_project_id? - @error_to_client(id:mesg.id, error:"src_project_id must be defined") - return - if not mesg.target_project_id? - @error_to_client(id:mesg.id, error:"target_project_id must be defined") - return - if not mesg.src_path? - @error_to_client(id:mesg.id, error:"src_path must be defined") - return - project = undefined - async.series([ - (cb) => - # ensure user can write to the target project - access.user_has_write_access_to_project - database : @database - project_id : mesg.target_project_id - account_id : @account_id - account_groups : @groups - cb : (err, result) => - if err - cb(err) - else if not result - cb("user must have write access to target project #{mesg.target_project_id}") - else - cb() - (cb) => - # Obviously, no need to check write access about the source project, - # since we are only granting access to public files. This function - # should ensure that the path is public: - @get_public_project - project_id : mesg.src_project_id - path : mesg.src_path - cb : (err, x) => - project = x - cb(err) - (cb) => - try - await project.copyPath - path : mesg.src_path - target_project_id : mesg.target_project_id - target_path : mesg.target_path - overwrite_newer : mesg.overwrite_newer - delete_missing : mesg.delete_missing - timeout : mesg.timeout - backup : mesg.backup - public : true - wait_until_done : true - cb() - catch err - cb(err) - ], (err) => - if err - @error_to_client(id:mesg.id, error:err) - else - @push_to_client(message.success(id:mesg.id)) - ) - - - ### - Stripe-integration billing code - ### - handle_stripe_mesg: (mesg) => - try - await @_stripe_client ?= new StripeClient(@) - catch err - @error_to_client(id:mesg.id, error:"${err}") - return - @_stripe_client.handle_mesg(mesg) - - mesg_stripe_get_customer: (mesg) => - @handle_stripe_mesg(mesg) - - mesg_stripe_create_source: (mesg) => - @handle_stripe_mesg(mesg) - - mesg_stripe_delete_source: (mesg) => - @handle_stripe_mesg(mesg) - - mesg_stripe_set_default_source: (mesg) => - @handle_stripe_mesg(mesg) - - mesg_stripe_update_source: (mesg) => - @handle_stripe_mesg(mesg) - - mesg_stripe_create_subscription: (mesg) => - @handle_stripe_mesg(mesg) - - mesg_stripe_cancel_subscription: (mesg) => - @handle_stripe_mesg(mesg) - - mesg_stripe_update_subscription: (mesg) => - @handle_stripe_mesg(mesg) - - mesg_stripe_get_subscriptions: (mesg) => - @handle_stripe_mesg(mesg) - - mesg_stripe_get_coupon: (mesg) => - @handle_stripe_mesg(mesg) - - mesg_stripe_get_charges: (mesg) => - @handle_stripe_mesg(mesg) - - mesg_stripe_get_invoices: (mesg) => - @handle_stripe_mesg(mesg) - - mesg_stripe_admin_create_invoice_item: (mesg) => - @handle_stripe_mesg(mesg) - - mesg_get_available_upgrades: (mesg) => - mesg.event = 'stripe_get_available_upgrades' # for backward compat - @handle_stripe_mesg(mesg) - - mesg_remove_all_upgrades: (mesg) => - mesg.event = 'stripe_remove_all_upgrades' # for backward compat - @handle_stripe_mesg(mesg) - - mesg_purchase_license: (mesg) => - try - await @_stripe_client ?= new StripeClient(@) - resp = await purchase_license(@account_id, mesg.info) - @push_to_client(message.purchase_license_resp(id:mesg.id, resp:resp)) - catch err - @error_to_client(id:mesg.id, error:err.toString()) - - # END stripe-related functionality - - _check_project_access: (project_id, cb) => - if not @account_id? - cb('you must be signed in to access project') - return - if not misc.is_valid_uuid_string(project_id) - cb('project_id must be specified and valid') - return - access.user_has_write_access_to_project - database : @database - project_id : project_id - account_groups : @groups - account_id : @account_id - cb : (err, result) => - if err - cb(err) - else if not result - cb("must have write access") - else - cb() - - _check_syncdoc_access: (string_id, cb) => - if not @account_id? - cb('you must be signed in to access syncdoc') - return - if not typeof string_id == 'string' and string_id.length == 40 - cb('string_id must be specified and valid') - return - @database._query - query : "SELECT project_id FROM syncstrings" - where : {"string_id = $::CHAR(40)" : string_id} - cb : (err, results) => - if err - cb(err) - else if results.rows.length != 1 - cb("no such syncdoc") - else - project_id = results.rows[0].project_id - @_check_project_access(project_id, cb) - - mesg_disconnect_from_project: (mesg) => - dbg = @dbg('mesg_disconnect_from_project') - @_check_project_access mesg.project_id, (err) => - if err - dbg("failed -- #{err}") - @error_to_client(id:mesg.id, error:"unable to disconnect from project #{mesg.project_id} -- #{err}") - else - local_hub_connection.disconnect_from_project(mesg.project_id) - @push_to_client(message.success(id:mesg.id)) - - mesg_get_syncdoc_history: (mesg) => - dbg = @dbg('mesg_syncdoc_history') - try - dbg("checking conditions") - # this raises an error if user does not have access - await callback(@_check_syncdoc_access, mesg.string_id) - # get the history - history = await @database.syncdoc_history_async(mesg.string_id, mesg.patches) - dbg("success!") - @push_to_client(message.syncdoc_history(id:mesg.id, history:history)) - catch err - dbg("failed -- #{err}") - @error_to_client(id:mesg.id, error:"unable to get syncdoc history for string_id #{mesg.string_id} -- #{err}") - - mesg_admin_reset_password: (mesg) => - dbg = @dbg("mesg_reset_password") - dbg(mesg.email_address) - try - if not misc.is_valid_email_address(mesg.email_address) - throw Error("invalid email address") - await callback(@assert_user_is_in_group, 'admin') - if not await callback2(@database.account_exists, {email_address : mesg.email_address}) - throw Error("no such account with email #{mesg.email_address}") - # We now know that there is an account with this email address. - # put entry in the password_reset uuid:value table with ttl of 24 hours. - # NOTE: when users request their own reset, the ttl is 1 hour, but when we - # as admins send one manually, they typically need more time, so 1 day instead. - # We used 8 hours for a while and it is often not enough time. - id = await callback2(@database.set_password_reset, {email_address : mesg.email_address, ttl:24*60*60}); - mesg.link = "/auth/password-reset/#{id}" - @push_to_client(mesg) - catch err - dbg("failed -- #{err}") - @error_to_client(id:mesg.id, error:"#{err}") - - - # These are deprecated. Not the best approach. - mesg_openai_embeddings_search: (mesg) => - @error_to_client(id:mesg.id, error:"openai_embeddings_search is DEPRECATED") - - mesg_openai_embeddings_save: (mesg) => - @error_to_client(id:mesg.id, error:"openai_embeddings_save is DEPRECATED") - - mesg_openai_embeddings_remove: (mesg) => - @error_to_client(id:mesg.id, error:"openai_embeddings_remove is DEPRECATED") - - mesg_jupyter_execute: (mesg) => - dbg = @dbg("mesg_jupyter_execute") - dbg(mesg.text) - if not @account_id? - @error_to_client(id:mesg.id, error:"not signed in") - return - try - resp = await jupyter_execute(mesg) - resp.id = mesg.id - @push_to_client(message.jupyter_execute_response(resp)) - catch err - dbg("failed -- #{err}") - @error_to_client(id:mesg.id, error:"#{err}") - - mesg_jupyter_kernels: (mesg) => - dbg = @dbg("mesg_jupyter_kernels") - dbg(mesg.text) - try - @push_to_client(message.jupyter_kernels(id:mesg.id, kernels:await jupyter_kernels({project_id:mesg.project_id, account_id:@account_id}))) - catch err - dbg("failed -- #{err}") - @error_to_client(id:mesg.id, error:"#{err}") - diff --git a/src/packages/hub/clients.ts b/src/packages/hub/clients.ts deleted file mode 100644 index a7406dc824..0000000000 --- a/src/packages/hub/clients.ts +++ /dev/null @@ -1,14 +0,0 @@ -//######################################################################## -// This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. -// License: MS-RSL – see LICENSE.md for details -//######################################################################## - -const clients: { [id: string]: any } = {}; - -export function getClients() { - return clients; -} - -export function pushToClient(mesg: { client_id: string }): void { - clients[mesg.client_id]?.push_to_client(mesg); -} diff --git a/src/packages/hub/hub.ts b/src/packages/hub/hub.ts index 5e799a4c85..c80d0e269b 100644 --- a/src/packages/hub/hub.ts +++ b/src/packages/hub/hub.ts @@ -16,7 +16,7 @@ import basePath from "@cocalc/backend/base-path"; import { pghost as DEFAULT_DB_HOST, pgdatabase as DEFAULT_DB_NAME, - pguser as DEFAULT_DB_USER, + pguser as DEFAULT_DB_USER, } from "@cocalc/backend/data"; import { trimLogFileSize } from "@cocalc/backend/logger"; import port from "@cocalc/backend/port"; @@ -35,22 +35,20 @@ import initPurchasesMaintenanceLoop from "@cocalc/server/purchases/maintenance"; import initSalesloftMaintenance from "@cocalc/server/salesloft/init"; import { stripe_sync } from "@cocalc/server/stripe/sync"; import { callback2, retry_until_success } from "@cocalc/util/async-utils"; -import { getClients } from "./clients"; import { set_agent_endpoint } from "./health-checks"; import { start as startHubRegister } from "./hub_register"; import { getLogger } from "./logger"; import initDatabase, { database } from "./servers/database"; import initExpressApp from "./servers/express-app"; import { - loadNatsConfiguration, - initNatsDatabaseServer, - initNatsChangefeedServer, - initNatsTieredStorage, - initNatsServer, -} from "@cocalc/server/nats"; + loadConatConfiguration, + initConatChangefeedServer, + initConatApi, + initConatPersist, +} from "@cocalc/server/conat"; +import { initConatServer } from "@cocalc/server/conat/socketio"; + import initHttpRedirect from "./servers/http-redirect"; -import initPrimus from "./servers/primus"; -import initVersionServer from "./servers/version"; const MetricsRecorder = require("./metrics-recorder"); // import * as MetricsRecorder from "./metrics-recorder"; @@ -61,13 +59,8 @@ const logger = getLogger("hub"); let program: { [option: string]: any } = {}; export { program }; -// How frequently to register with the database that this hub is up and running, -// and also report number of connected clients. const REGISTER_INTERVAL_S = 20; -// the jsmap of connected clients -const clients = getClients(); - async function reset_password(email_address: string): Promise { try { await callback2(database.reset_password, { email_address }); @@ -177,7 +170,7 @@ async function startServer(): Promise { // used for nextjs hot module reloading dev server process.env["COCALC_MODE"] = program.mode; - if (program.mode != "kucalc" && program.websocketServer) { + if (program.mode != "kucalc" && program.conatServer) { // We handle idle timeout of projects. // This can be disabled via COCALC_NO_IDLE_TIMEOUT. // This only uses the admin-configurable settings field of projects @@ -185,31 +178,24 @@ async function startServer(): Promise { initIdleTimeout(projectControl); } - // all configuration MUST load nats configuration. This loads - // credentials to use nats from the database, and is needed - // by many things. - await loadNatsConfiguration(); + // This loads from the database credentials to use Conat. + await loadConatConfiguration(); - if (program.natsServer) { - await initNatsServer(); - } - if (program.natsDatabaseServer) { - await initNatsDatabaseServer(); - } - if (program.natsChangefeedServer) { - await initNatsChangefeedServer(); + if (program.conatCore) { + // launch standalone socketio websocket server (no http server) + await initConatServer(); } - if (program.natsTieredStorage) { - // currently there must be exactly ONE of these, running on the same - // node as the nats-server. E.g., for development it's just part of the server. - await initNatsTieredStorage(); + + if (program.conatApi || program.conatServer) { + await initConatApi(); + await initConatChangefeedServer(); } - if (program.websocketServer) { - // Initialize the version server -- must happen after updating schema - // (for first ever run). - await initVersionServer(); + if (program.conatPersist || program.conatServer) { + await initConatPersist(); + } + if (program.conatServer) { if (program.mode == "single-user" && process.env.USER == "user") { // Definitely in dev mode, probably on cocalc.com in a project, so we kill // all the running projects when starting the hub: @@ -240,53 +226,43 @@ async function startServer(): Promise { } } - const { router, httpServer } = await initExpressApp({ - isPersonal: program.personal, - projectControl, - proxyServer: !!program.proxyServer, - nextServer: !!program.nextServer, - cert: program.httpsCert, - key: program.httpsKey, - listenersHack: - program.mode == "single-user" && - program.proxyServer && - program.nextServer && - program.websocketServer && - process.env["NODE_ENV"] == "development", - }); - - //initNatsServer(); - - // The express app create via initExpressApp above **assumes** that init_passport is done - // or complains a lot. This is obviously not really necessary, but we leave it for now. - await callback2(init_passport, { - router, - database, - host: program.hostname, - }); - - logger.info(`starting webserver listening on ${program.hostname}:${port}`); - await callback(httpServer.listen.bind(httpServer), port, program.hostname); - - if (port == 443 && program.httpsCert && program.httpsKey) { - // also start a redirect from port 80 to port 443. - await initHttpRedirect(program.hostname); - } + if ( + program.conatServer || + program.proxyServer || + program.nextServer || + program.conatApi + ) { + const { router, httpServer } = await initExpressApp({ + isPersonal: program.personal, + projectControl, + conatServer: !!program.conatServer, + proxyServer: true, // always + nextServer: !!program.nextServer, + cert: program.httpsCert, + key: program.httpsKey, + listenersHack: + program.mode == "single-user" && + program.proxyServer && + program.nextServer && + process.env["NODE_ENV"] == "development", + }); - if (program.websocketServer) { - logger.info("initializing primus websocket server"); - initPrimus({ - httpServer, + // The express app create via initExpressApp above **assumes** that init_passport is done + // or complains a lot. This is obviously not really necessary, but we leave it for now. + await callback2(init_passport, { router, - projectControl, - clients, + database, host: program.hostname, - port, - isPersonal: program.personal, }); - } - if (program.websocketServer || program.proxyServer || program.nextServer) { + logger.info(`starting webserver listening on ${program.hostname}:${port}`); + await callback(httpServer.listen.bind(httpServer), port, program.hostname); + + if (port == 443 && program.httpsCert && program.httpsKey) { + // also start a redirect from port 80 to port 443. + await initHttpRedirect(program.hostname); + } + logger.info( "Starting registering periodically with the database and updating a health check...", ); @@ -295,7 +271,6 @@ async function startServer(): Promise { // also confirms that database is working. await callback2(startHubRegister, { database, - clients, host: program.hostname, port, interval_s: REGISTER_INTERVAL_S, @@ -315,7 +290,7 @@ async function startServer(): Promise { console.log(msg); if ( - program.websocketServer && + program.conatServer && program.nextServer && process.env["NODE_ENV"] != "production" ) { @@ -368,12 +343,14 @@ async function startServer(): Promise { // more than once per minute. const errorReportCache = new TTLCache({ ttl: 60 * 1000 }); +// note -- we show the error twice in these, one in backticks, since sometimes +// that works better. function addErrorListeners(uncaught_exception_total) { process.addListener("uncaughtException", function (err) { logger.error( "BUG ****************************************************************************", ); - logger.error("Uncaught exception: " + err); + logger.error("Uncaught exception: " + err, ` ${err}`); console.error(err.stack); logger.error(err.stack); logger.error( @@ -393,7 +370,13 @@ function addErrorListeners(uncaught_exception_total) { "BUG UNHANDLED REJECTION *********************************************************", ); console.error(p, reason); // strangely sometimes logger.error can't actually show the traceback... - logger.error("Unhandled Rejection at:", p, "reason:", reason); + logger.error( + "Unhandled Rejection at:", + p, + "reason:", + reason, + ` : ${p} -- ${reason}`, + ); logger.error( "BUG UNHANDLED REJECTION *********************************************************", ); @@ -426,18 +409,21 @@ async function main(): Promise { "--all", "runs all of the servers: websocket, proxy, next (so you don't have to pass all those opts separately), and also mentions updator and updates db schema on startup; use this in situations where there is a single hub that serves everything (instead of a microservice situation like kucalc)", ) - .option("--websocket-server", "run a websocket server in this process") .option( - "--nats-server", - "run a hub that servers standard nats microservices, e.g., LLM's, authentication, etc. There should be at least one of these.", + "--conat-server", + "run a hub that provides a single-core conat server (socketio), api, and persistence, along with an http server. This is for dev and small deployments of cocalc (and if given, do not bother with --conat-[core|api|persist] below.)", + ) + .option( + "--conat-core", + "run a hub that provides the core conat communication layer server over a websocket (but not http server). If you run more than one of these at once they MUST be configured to use valkey, or messages won't get transmitted between them.", ) .option( - "--nats-database-server", - "run NATS microservice to provide access (including changefeeds) to the database", + "--conat-api", + "run a hub that connect to conat-core and provides the standard conat API services, e.g., basic api, LLM's, changefeeds, http file upload/download, etc. There must be at least one of these. You can increase or decrease the number of these servers with no coordination needed.", ) .option( - "--nats-changefeed-server", - "run NATS microservice to provide postgres based changefeeds; there must be AT LEAST one of these.", + "--conat-persist", + "run a hub that connects to conat-core and provides persistence for streams (e.g., key for sync editing). There must be at least one of these, and they need access to common shared disk to store sqlite files. Only one server uses a given sqlite file at a time. You can increase or decrease the number of these servers with no coordination needed.", ) .option("--proxy-server", "run a proxy server in this process") .option( @@ -536,10 +522,7 @@ async function main(): Promise { } } if (program.all) { - program.websocketServer = - program.natsServer = - program.natsChangefeedServer = - program.natsTieredStorage = + program.conatServer = program.proxyServer = program.nextServer = program.mentions = diff --git a/src/packages/hub/hub_register.ts b/src/packages/hub/hub_register.ts index 019193b7db..26e84bdefd 100644 --- a/src/packages/hub/hub_register.ts +++ b/src/packages/hub/hub_register.ts @@ -3,7 +3,7 @@ * License: MS-RSL – see LICENSE.md for details */ -// Hub Registration (recording number of clients) +// Hub Registration const winston = require("./logger").getLogger("hub"); import * as misc from "@cocalc/util/misc"; @@ -17,18 +17,6 @@ let the_database: PostgreSQL | undefined = undefined; let the_host: string | undefined = undefined; let the_port: number | undefined = undefined; let the_interval: number | undefined = undefined; // seconds -let the_clients: any = {}; - -const number_of_clients = () => misc.len(the_clients); - -function _number_of_clients() { - if (the_database == null) { - throw new Error("database not yet set"); - } - return number_of_clients(); -} - -export { _number_of_clients as number_of_clients }; function register_hub(cb) { winston.debug("register_hub"); @@ -63,13 +51,13 @@ function register_hub(cb) { winston.debug("register_hub -- doing db query"); if (the_host == null || the_port == null || the_interval == null) { throw new Error( - "the_host, the_port, and the_interval must be set before registering this hub" + "the_host, the_port, and the_interval must be set before registering this hub", ); } the_database.register_hub({ host: the_host, port: the_port, - clients: number_of_clients(), + clients: 0, // TODO ttl: 3 * the_interval, cb(err) { if (err) { @@ -92,7 +80,6 @@ export { _database_is_working as database_is_working }; interface Opts { database: PostgreSQL; - clients: any; host: string; port: number; interval_s: number; @@ -102,7 +89,6 @@ interface Opts { export function start(opts: Opts): void { opts = defaults(opts, { database: required, - clients: required, host: required, port: required, interval_s: required, @@ -115,7 +101,6 @@ export function start(opts: Opts): void { started = true; } the_database = opts.database; - the_clients = opts.clients; the_host = opts.host; the_port = opts.port; the_interval = opts.interval_s; diff --git a/src/packages/hub/local_hub_connection.coffee b/src/packages/hub/local_hub_connection.coffee index 99f822509d..cb48d3fef7 100644 --- a/src/packages/hub/local_hub_connection.coffee +++ b/src/packages/hub/local_hub_connection.coffee @@ -30,7 +30,6 @@ misc = require('@cocalc/util/misc') {defaults, required} = misc blobs = require('./blobs') -clients = require('./clients') # Blobs (e.g., files dynamically appearing as output in worksheets) are kept for this # many seconds before being discarded. If the worksheet is saved (e.g., by a user's autosave), @@ -315,16 +314,6 @@ class LocalHub # use the function "new_local_hub" above; do not construct this d throw Error("project does NOT have access to this syncdoc") return # everything is fine. - mesg_get_syncdoc_history: (mesg, write_mesg) => - try - # this raises an error if user does not have access - await @check_syncdoc_access(mesg.string_id) - # get the history - history = await @database.syncdoc_history_async(mesg.string_id, mesg.patches) - write_mesg(message.syncdoc_history(id:mesg.id, history:history)) - catch err - write_mesg(message.error(id:mesg.id, error:"unable to get syncdoc history for string_id #{mesg.string_id} -- #{err}")) - # # end project query support code # @@ -382,9 +371,7 @@ class LocalHub # use the function "new_local_hub" above; do not construct this d # know the client's id, which is a random uuid, assigned each time the user connects. # It obviously is known to the local hub -- but if the user has connected to the local # hub then they should be allowed to receive messages. - # NOTE: this should be possible to deprecate, because the clients all connect via - # a websocket directly to the project. - clients.pushToClient(mesg) + # *DEPRECATED* return if mesg.event == 'version' @local_hub_version(mesg.version) @@ -407,8 +394,6 @@ class LocalHub # use the function "new_local_hub" above; do not construct this d @mesg_query(mesg, write_mesg) when 'query_cancel' @mesg_query_cancel(mesg, write_mesg) - when 'get_syncdoc_history' - @mesg_get_syncdoc_history(mesg, write_mesg) when 'file_written_to_project' # ignore -- don't care; this is going away return diff --git a/src/packages/hub/package.json b/src/packages/hub/package.json index 206f507d76..0fcfe1a646 100644 --- a/src/packages/hub/package.json +++ b/src/packages/hub/package.json @@ -10,25 +10,17 @@ "@cocalc/assets": "workspace:*", "@cocalc/backend": "workspace:*", "@cocalc/cdn": "workspace:*", + "@cocalc/conat": "workspace:*", "@cocalc/database": "workspace:*", - "@cocalc/frontend": "workspace:*", "@cocalc/hub": "workspace:*", - "@cocalc/nats": "workspace:*", "@cocalc/next": "workspace:*", "@cocalc/server": "workspace:*", "@cocalc/static": "workspace:*", "@cocalc/util": "workspace:*", - "@isaacs/ttlcache": "^1.2.1", - "@passport-next/passport-google-oauth2": "^1.0.0", - "@passport-next/passport-oauth2": "^2.1.4", + "@isaacs/ttlcache": "^1.4.1", "@types/formidable": "^3.4.5", - "@types/primus": "^7.3.9", - "@types/react": "^18.3.10", - "@types/uuid": "^8.3.1", "async": "^1.5.2", "awaiting": "^3.0.0", - "basic-auth": "^2.0.1", - "bindings": "^1.3.0", "blocked": "^1.1.0", "commander": "^7.2.0", "compression": "^1.7.4", @@ -36,67 +28,45 @@ "cookies": "^0.8.0", "cors": "^2.8.5", "debug": "^4.4.0", - "escape-html": "^1.0.3", "express": "^4.21.2", "formidable": "^3.5.4", - "http-proxy-3": "^1.20.0", - "immutable": "^4.3.0", - "jquery": "^3.6.0", - "json-stable-stringify": "^1.0.1", + "http-proxy-3": "^1.20.5", "lodash": "^4.17.21", "lru-cache": "^7.18.3", - "mime": "^1.3.4", "mime-types": "^2.1.35", - "mkdirp": "^1.0.4", "ms": "2.1.2", - "nats": "^2.29.3", - "next": "14.2.28", "parse-domain": "^5.0.0", - "passport": "^0.6.0", - "password-hash": "^1.2.2", - "primus": "^8.0.9", "prom-client": "^13.0.0", "random-key": "^0.3.2", "react": "^18.3.1", - "react-dom": "^18.3.1", - "read": "^1.0.7", - "require-reload": "^0.2.2", - "safe-json-stringify": "^1.2.0", - "serve-index": "^1.9.1", - "sql-string-escape": "^1.1.6", - "temp": "^0.9.4", "uglify-js": "^3.14.1", "underscore": "^1.12.1", "uuid": "^8.3.2", "validator": "^13.6.0", "webpack-dev-middleware": "^7.4.2", - "webpack-hot-middleware": "^2.26.1", - "ws": "^8.18.0" + "webpack-hot-middleware": "^2.26.1" }, "devDependenicesNotes": "For license and size reasons, we make @cocalc/crm a dev dependency so it is NOT installed unless explicitly installed as a separate step.", "devDependencies": { "@types/express": "^4.17.21", "@types/node": "^18.16.14", - "@types/passport": "^1.0.9", - "@types/react-dom": "^18.3.0", - "coffeescript": "^2.5.1", - "expect": "^26.6.2", - "node-cjsx": "^2.0.0", - "should": "^7.1.1", - "sinon": "^4.5.0" + "coffeescript": "^2.5.1" }, "scripts": { "preinstall": "npx only-allow pnpm", - "build": "tsc --build && coffee -m -c -o dist/ .", - "hub-project-dev-nobuild": "unset DATA COCALC_ROOT BASE_PATH && export DEBUG=${DEBUG:='cocalc:*,-cocalc:silly:*'} && export PGHOST=${PGHOST:=`realpath $INIT_CWD/../../data/postgres/socket`} && PGUSER='smc' NODE_ENV=${NODE_ENV:=development} NODE_OPTIONS='--max_old_space_size=8000 --trace-warnings --enable-source-maps --inspect' cocalc-hub-server --mode=single-user --all --hostname=0.0.0.0", - "hub-personal": "unset DATA COCALC_ROOT BASE_PATH && export DEBUG=${DEBUG:='cocalc:*,-cocalc:silly:*'} && export PGHOST=${PGHOST:=`realpath $INIT_CWD/../../data/postgres/socket`} && PGUSER='smc' NODE_ENV=${NODE_ENV:=development} NODE_OPTIONS='--max_old_space_size=8000 --trace-warnings --enable-source-maps --inspect' cocalc-hub-server --mode=single-user --all --hostname=0.0.0.0 --personal", + "clean": "rm -rf node_modules dist", + "build": "tsc && coffee -m -c -o dist/ .", + "hub-project-dev-nobuild": "unset DATA COCALC_ROOT && export DEBUG=${DEBUG:='cocalc:*,-cocalc:silly:*'} && export PGHOST=${PGHOST:=`realpath $INIT_CWD/../../data/postgres/socket`} && PGUSER='smc' NODE_ENV=${NODE_ENV:=development} NODE_OPTIONS='--max_old_space_size=8000 --trace-warnings --enable-source-maps --inspect' cocalc-hub-server --mode=single-user --all --hostname=0.0.0.0", + "hub-personal": "unset DATA COCALC_ROOT && export DEBUG=${DEBUG:='cocalc:*,-cocalc:silly:*'} && export PGHOST=${PGHOST:=`realpath $INIT_CWD/../../data/postgres/socket`} && PGUSER='smc' NODE_ENV=${NODE_ENV:=development} NODE_OPTIONS='--max_old_space_size=8000 --trace-warnings --enable-source-maps --inspect' cocalc-hub-server --mode=single-user --all --hostname=0.0.0.0 --personal", "hub-project-dev": "pnpm build && NODE_OPTIONS='--inspect' pnpm hub-project-dev-nobuild", - "hub-project-prod-nobuild": "unset DATA COCALC_ROOT BASE_PATH && export DEBUG=${DEBUG:='cocalc:*,-cocalc:silly:*'} && export PGHOST=${PGHOST:=`realpath $INIT_CWD/../../data/postgres/socket`} && PGUSER='smc' NODE_ENV=production NODE_OPTIONS='--max_old_space_size=8000 --enable-source-maps' cocalc-hub-server --mode=single-user --all --hostname=0.0.0.0", + "hub-project-prod-nobuild": "unset DATA COCALC_ROOT && export DEBUG=${DEBUG:='cocalc:*,-cocalc:silly:*'} && export PGHOST=${PGHOST:=`realpath $INIT_CWD/../../data/postgres/socket`} && PGUSER='smc' NODE_ENV=production NODE_OPTIONS='--max_old_space_size=8000 --enable-source-maps' cocalc-hub-server --mode=single-user --all --hostname=0.0.0.0", + "hub-project-prod-ssl": "unset DATA COCALC_ROOT && export CONAT_SERVER=https://localhost:$PORT && export DEBUG=${DEBUG:='cocalc:*,-cocalc:silly:*'} && export PGHOST=${PGHOST:=`realpath $INIT_CWD/../../data/postgres/socket`} && PGUSER='smc' NODE_ENV=production NODE_OPTIONS='--max_old_space_size=8000 --enable-source-maps' cocalc-hub-server --mode=single-user --all --hostname=0.0.0.0 --https-key=$INIT_CWD/../../data/secrets/cert/key.pem --https-cert=$INIT_CWD/../../data/secrets/cert/cert.pem", "hub-docker-dev": "export DEBUG=${DEBUG:='cocalc:*,-cocalc:silly:*'} && COCALC_DOCKER=true NODE_ENV=development PROJECTS=/projects/[project_id] PORT=443 NODE_OPTIONS='--max_old_space_size=8000 --enable-source-maps --trace-warnings --inspect' cocalc-hub-server --mode=multi-user --all --hostname=0.0.0.0 --https-key=/projects/conf/cert/key.pem --https-cert=/projects/conf/cert/cert.pem", "hub-docker-prod": "export DEBUG=${DEBUG:='cocalc:*,-cocalc:silly:*'} && COCALC_DOCKER=true NODE_ENV=production PROJECTS=/projects/[project_id] PORT=443 NODE_OPTIONS='--max_old_space_size=8000 --enable-source-maps' cocalc-hub-server --mode=multi-user --all --hostname=0.0.0.0 --https-key=/projects/conf/cert/key.pem --https-cert=/projects/conf/cert/cert.pem", "hub-docker-prod-nossl": "export DEBUG=${DEBUG:='cocalc:*,-cocalc:silly:*'} && COCALC_DOCKER=true NODE_ENV=production PROJECTS=/projects/[project_id] PORT=80 NODE_OPTIONS='--max_old_space_size=8000 --enable-source-maps' cocalc-hub-server --mode=multi-user --all --hostname=0.0.0.0", "tsc": "tsc --watch --pretty --preserveWatchOutput", "test": "jest dist/", + "depcheck": "pnpx depcheck | grep -Ev '\\.coffee|coffee$'", "prepublishOnly": "test" }, "repository": { diff --git a/src/packages/hub/primus-client.ts b/src/packages/hub/primus-client.ts deleted file mode 100644 index 886cf575f6..0000000000 --- a/src/packages/hub/primus-client.ts +++ /dev/null @@ -1,18 +0,0 @@ -/* add endpoint that serves the primus client js code. */ - -const UglifyJS = require("uglify-js"); - -export default function setupPrimusClient(router, primus): void { - const primus_js = primus.library(); - const primus_min_js: string = UglifyJS.minify(primus_js).code; - router.get("/primus.js", (_, res) => { - res.header("Content-Type", "text/javascript"); - res.header("Cache-Control", `private, max-age=${60 * 60}, must-revalidate`); - res.send(primus_js); - }); - router.get("/primus.min.js", (_, res) => { - res.header("Content-Type", "text/javascript"); - res.header("Cache-Control", `private, max-age=${60 * 60}, must-revalidate`); - res.send(primus_min_js); - }); -} diff --git a/src/packages/hub/proxy/handle-request.ts b/src/packages/hub/proxy/handle-request.ts index c45e0da5da..8f38785647 100644 --- a/src/packages/hub/proxy/handle-request.ts +++ b/src/packages/hub/proxy/handle-request.ts @@ -10,7 +10,7 @@ import { stripBasePath } from "./util"; import { ProjectControlFunction } from "@cocalc/server/projects/control"; import siteUrl from "@cocalc/database/settings/site-url"; import { parseReq } from "./parse"; -import { readFile as readProjectFile } from "@cocalc/nats/files/read"; +import { readFile as readProjectFile } from "@cocalc/conat/files/read"; import { path_split } from "@cocalc/util/misc"; import { once } from "@cocalc/util/async-utils"; import hasAccess from "./check-for-access-to-project"; @@ -84,7 +84,7 @@ export default function init({ projectControl, isPersonal }: Options) { // TODO: parseReq is called again in getTarget so need to refactor... const { type, project_id } = parsed; if (type == "files") { - dbg("handling the request via nats"); + dbg("handling the request via conat file streaming"); if ( !(await hasAccess({ project_id, @@ -103,7 +103,7 @@ export default function init({ projectControl, isPersonal }: Options) { j = url.length; } const path = decodeURIComponent(url.slice(i + "files/".length, j)); - dbg("NATs: get", { project_id, path, compute_server_id, url }); + dbg("conat: get file", { project_id, path, compute_server_id, url }); const fileName = path_split(path).tail; const contentType = mime.lookup(fileName); if ( diff --git a/src/packages/hub/proxy/handle-upgrade.ts b/src/packages/hub/proxy/handle-upgrade.ts index 4a7d3d3050..027e37fbc2 100644 --- a/src/packages/hub/proxy/handle-upgrade.ts +++ b/src/packages/hub/proxy/handle-upgrade.ts @@ -3,18 +3,18 @@ import { createProxyServer, type ProxyServer } from "http-proxy-3"; import LRU from "lru-cache"; import { getEventListeners } from "node:events"; - import getLogger from "@cocalc/hub/logger"; import stripRememberMeCookie from "./strip-remember-me-cookie"; import { getTarget } from "./target"; import { stripBasePath } from "./util"; import { versionCheckFails } from "./version"; -import { proxyNatsWebsocket } from "@cocalc/hub/servers/nats"; +import { proxyConatWebsocket } from "./proxy-conat"; +import basePath from "@cocalc/backend/base-path"; const logger = getLogger("proxy:handle-upgrade"); export default function init( - { projectControl, isPersonal, httpServer, listenersHack }, + { projectControl, isPersonal, httpServer, listenersHack, proxyConat }, proxy_regexp: string, ) { const cache = new LRU({ @@ -25,24 +25,33 @@ export default function init( const re = new RegExp(proxy_regexp); async function handleProxyUpgradeRequest(req, socket, head): Promise { + if (proxyConat) { + const u = new URL(req.url, "http://cocalc.com"); + let pathname = u.pathname; + if (basePath.length > 1) { + pathname = pathname.slice(basePath.length); + } + if (pathname == "/conat/") { + proxyConatWebsocket(req, socket, head); + return; + } + } + + if (!req.url.match(re)) { + // something else (e.g., the socket.io server) is handling this websocket; + // we do NOT mess with anything in this case + return; + } + socket.on("error", (err) => { // server will crash sometimes without this: logger.debug("WARNING -- websocket socket error", err); }); + const dbg = (...args) => { logger.silly(req.url, ...args); }; dbg("got upgrade request from url=", req.url); - const url = stripBasePath(req.url); - - if (url == "/nats") { - proxyNatsWebsocket(req, socket, head); - return; - } - - if (!req.url.match(re)) { - throw Error(`url=${req.url} does not support upgrade`); - } // Check that minimum version requirement is satisfied (this is in the header). // This is to have a way to stop buggy clients from causing trouble. It's a purely @@ -62,6 +71,7 @@ export default function init( } dbg("calling getTarget"); + const url = stripBasePath(req.url); const { host, port, internal_url } = await getTarget({ url, isPersonal, @@ -146,6 +156,10 @@ export default function init( // NOTE: I had to do something similar that is in packages/next/lib/init.js, // and is NOT a hack. That technique could probably be used to fix this properly. + // NOTE2: It's May 2025, and I basically don't use HMR anymore and just refresh + // my page, since dealing with this is so painful. Also rspack is superfast and + // refresh is fast, so HMR feels less necessary. Finally, frequently any dev work + // I do requires a page refresh anyways. let listeners: any[] = []; handler = async (req, socket, head) => { diff --git a/src/packages/hub/proxy/index.ts b/src/packages/hub/proxy/index.ts index a4743845d9..f50f188909 100644 --- a/src/packages/hub/proxy/index.ts +++ b/src/packages/hub/proxy/index.ts @@ -18,6 +18,7 @@ interface Options { projectControl: ProjectControlFunction; // controls projects (aka "compute server") isPersonal: boolean; // if true, disables all access controls listenersHack: boolean; + proxyConat: boolean; } export default function init(opts: Options) { diff --git a/src/packages/hub/proxy/proxy-conat.ts b/src/packages/hub/proxy/proxy-conat.ts new file mode 100644 index 0000000000..0ee8c9e439 --- /dev/null +++ b/src/packages/hub/proxy/proxy-conat.ts @@ -0,0 +1,42 @@ +/* +Conat WebSocket proxy -- this primarily just directly proxied the conats +socketio websocket server, so outside browsers can connect to it. +So far I'm only using this for testing, but it could be useful in a non-kubernetes +setting, where we need certain types of scalability. +*/ + +import { createProxyServer, type ProxyServer } from "http-proxy-3"; +import getLogger from "@cocalc/backend/logger"; +import { + conatServer as conatServer0, + conatClusterPort, +} from "@cocalc/backend/data"; +import basePath from "@cocalc/backend/base-path"; + +const logger = getLogger("hub:proxy-conat"); + +let proxy: ProxyServer | null = null; +export async function proxyConatWebsocket(req, socket, head) { + const conatServer = conatServer0 + ? conatServer0 + : `http://localhost:${conatClusterPort}${basePath.length > 1 ? basePath : ""}`; + const i = req.url.lastIndexOf("/conat"); + const target = conatServer + req.url.slice(i); + logger.debug(`conat proxy -- proxying a WEBSOCKET connection to ${target}`); + // todo -- allowing no cookie, since that's used by projects and compute servers! + // do NOT disable this until compute servers all set a cookie... which could be a long time. + if (proxy == null) { + // make the proxy server + proxy = createProxyServer({ + ws: true, + secure: false, + target, + }); + proxy.on("error", (err) => { + logger.debug(`WARNING: conat websocket proxy error -- ${err}`); + }); + } + + // connect the client's socket to conat via the proxy server: + proxy.ws(req, socket, head); +} diff --git a/src/packages/hub/proxy/strip-remember-me-cookie.ts b/src/packages/hub/proxy/strip-remember-me-cookie.ts index 5e582b9c19..802f30a064 100644 --- a/src/packages/hub/proxy/strip-remember-me-cookie.ts +++ b/src/packages/hub/proxy/strip-remember-me-cookie.ts @@ -16,20 +16,17 @@ export default function stripRememberMeCookie(cookie): { cookie: string; remember_me: string | undefined; // the value of the cookie we just stripped out. api_key: string | undefined; - nats_jwt: string | undefined; } { if (cookie == null) { return { cookie, remember_me: undefined, api_key: undefined, - nats_jwt: undefined, }; } else { const v: string[] = []; let remember_me: string | undefined = undefined; let api_key: string | undefined = undefined; - let nats_jwt: string | undefined = undefined; for (const c of cookie.split(";")) { const z = c.split("="); if (z[0].trim() == REMEMBER_ME_COOKIE_NAME) { @@ -43,6 +40,6 @@ export default function stripRememberMeCookie(cookie): { v.push(c); } } - return { cookie: v.join(";"), remember_me, api_key, nats_jwt }; + return { cookie: v.join(";"), remember_me, api_key }; } } diff --git a/src/packages/hub/servers/app/api.ts b/src/packages/hub/servers/app/api.ts deleted file mode 100644 index 41b9360b6c..0000000000 --- a/src/packages/hub/servers/app/api.ts +++ /dev/null @@ -1,62 +0,0 @@ -/* -The HTTP API, which works via POST requests. -*/ -import { Router } from "express"; -import * as express from "express"; -const { http_message_api_v1 } = require("@cocalc/hub/api/handler"); -import { callback2 } from "@cocalc/util/async-utils"; -import { getLogger } from "@cocalc/hub/logger"; -import { database } from "../database"; -import { ProjectControlFunction } from "@cocalc/server/projects/control"; -import { getApiKey } from "@cocalc/server/auth/api"; - -export default function init( - app_router: Router, - projectControl: ProjectControlFunction -) { - const logger = getLogger("api-server"); - logger.info("Initializing API server at /api/v1"); - - const router = Router(); - - // We need to parse POST requests. - // Note that this can conflict with what is in - // packages/project/servers/browser/http-server.ts - // thus breaking file uploads to projects, so be careful! - // That's why we make a new Router and it only applies - // to the /api/v1 routes. We raise the limit since the - // default is really tiny, and there is an api call to - // upload the contents of a file. - router.use(express.urlencoded({ extended: true, limit: "1mb" })); - router.use(express.json()); // To parse the incoming requests with JSON payloads - - router.post("/*", async (req, res) => { - let api_key; - try { - api_key = getApiKey(req); - } catch (err) { - res.status(400).send({ error: err.message }); - return; - } - const { body } = req; - const path = req.baseUrl; - const event = path.slice(path.lastIndexOf("/") + 1); - logger.debug(`event=${event}, body=${JSON.stringify(body)}`); - try { - const resp = await callback2(http_message_api_v1, { - event, - body, - api_key, - logger, - database, - projectControl, - ip_address: req.ip, - }); - res.send(resp); - } catch (err) { - res.status(400).send({ error: `${err}` }); // Bad Request - } - }); - - app_router.use("/api/v1/*", router); -} diff --git a/src/packages/hub/servers/app/metrics.ts b/src/packages/hub/servers/app/metrics.ts index 3224018585..c434a8384d 100644 --- a/src/packages/hub/servers/app/metrics.ts +++ b/src/packages/hub/servers/app/metrics.ts @@ -27,17 +27,8 @@ function metrics(req, res, next) { if (!req.path) { return; } - const pathSplit = req.path.split("/"); - // for API paths, we want to have data for each endpoint - const path_tail = pathSplit.slice(pathSplit.length - 3); - const is_api = path_tail[0] === "api" && path_tail[1] === "v1"; - let path; - if (is_api) { - path = path_tail.join("/"); - } else { - // for regular paths, we ignore the file - path = dirname(req.path).split("/").slice(0, 2).join("/"); - } + // for regular paths, we ignore the file + const path = dirname(req.path).split("/").slice(0, 2).join("/"); resFinished({ path, method: req.method, @@ -52,8 +43,8 @@ export function setupInstrumentation(router: Router) { } async function isEnabled(pool): Promise { - const {rows} = await pool.query( - "SELECT value FROM server_settings WHERE name='prometheus_metrics'" + const { rows } = await pool.query( + "SELECT value FROM server_settings WHERE name='prometheus_metrics'", ); const enabled = rows.length > 0 && rows[0].value == "yes"; log.info("isEnabled", enabled); diff --git a/src/packages/hub/servers/app/set-cookies.ts b/src/packages/hub/servers/app/set-cookies.ts deleted file mode 100644 index 79111bc97a..0000000000 --- a/src/packages/hub/servers/app/set-cookies.ts +++ /dev/null @@ -1,26 +0,0 @@ -import Cookies from "cookies"; -import { Router } from "express"; -import { getLogger } from "@cocalc/hub/logger"; -const { COOKIE_OPTIONS } = require("@cocalc/hub/client"); // import { COOKIE_OPTIONS } from "@cocalc/hub/client"; - -export default function init(router: Router) { - const winston = getLogger("set-cookie"); - - router.get("/cookies", (req, res) => { - if (req.query.set) { - // TODO: implement setting maxAge as part of query? not needed for now. - const maxAge = 1000 * 24 * 3600 * 30 * 6; // 6 months -- long is fine now since we support "sign out everywhere" ? - - winston.debug(`${req.query.set}=${req.query.value}`); - // The option { secure: true } is needed if SSL happens outside the hub; see - // https://github.com/pillarjs/cookies/issues/51#issuecomment-568182639 - // It basically tells the server to pretend the connection is secure, even though - // it's internal heuristic based on req says it is not secure. - const cookies = new Cookies(req, res, { secure: true }); - const conf = { ...COOKIE_OPTIONS, maxAge }; - winston.debug(`conf=${JSON.stringify(conf)}`); - cookies.set(req.query.set, req.query.value, conf); - } - res.end(); - }); -} diff --git a/src/packages/hub/servers/app/upload.ts b/src/packages/hub/servers/app/upload.ts index 95df3847cc..9ac22efaa8 100644 --- a/src/packages/hub/servers/app/upload.ts +++ b/src/packages/hub/servers/app/upload.ts @@ -25,7 +25,7 @@ import getAccount from "@cocalc/server/auth/get-account"; import isCollaborator from "@cocalc/server/projects/is-collaborator"; import formidable from "formidable"; import { PassThrough } from "node:stream"; -import { writeFile as writeFileToProject } from "@cocalc/nats/files/write"; +import { writeFile as writeFileToProject } from "@cocalc/conat/files/write"; import { join } from "path"; import { callback } from "awaiting"; diff --git a/src/packages/hub/servers/express-app.ts b/src/packages/hub/servers/express-app.ts index d2776f339d..bd7ce3e2cf 100644 --- a/src/packages/hub/servers/express-app.ts +++ b/src/packages/hub/servers/express-app.ts @@ -11,7 +11,6 @@ import { parse as parseURL } from "url"; import webpackDevMiddleware from "webpack-dev-middleware"; import webpackHotMiddleware from "webpack-hot-middleware"; import { path as WEBAPP_PATH } from "@cocalc/assets"; -import basePath from "@cocalc/backend/base-path"; import { path as CDN_PATH } from "@cocalc/cdn"; import vhostShare from "@cocalc/next/lib/share/virtual-hosts"; import { path as STATIC_PATH } from "@cocalc/static"; @@ -19,7 +18,6 @@ import { initAnalytics } from "../analytics"; import { setup_health_checks as setupHealthChecks } from "../health-checks"; import { getLogger } from "../logger"; import initProxy from "../proxy"; -import initAPI from "./app/api"; import initAppRedirect from "./app/app-redirect"; import initBlobUpload from "./app/blob-upload"; import initUpload from "./app/upload"; @@ -27,12 +25,13 @@ import initBlobs from "./app/blobs"; import initCustomize from "./app/customize"; import { initMetricsEndpoint, setupInstrumentation } from "./app/metrics"; import initNext from "./app/next"; -import initSetCookies from "./app/set-cookies"; import initStats from "./app/stats"; import { database } from "./database"; import initHttpServer from "./http"; import initRobots from "./robots"; -import { initNatsServer } from "./nats"; +import basePath from "@cocalc/backend/base-path"; +import { initConatServer } from "@cocalc/server/conat/socketio"; +import { conatClusterPort } from "@cocalc/backend/data"; // Used for longterm caching of files. This should be in units of seconds. const MAX_AGE = Math.round(ms("10 days") / 1000); @@ -43,6 +42,7 @@ interface Options { isPersonal: boolean; nextServer: boolean; proxyServer: boolean; + conatServer: boolean; cert?: string; key?: string; listenersHack: boolean; @@ -100,8 +100,6 @@ export default async function init(opts: Options): Promise<{ // setup the analytics.js endpoint await initAnalytics(router, database); - initAPI(router, opts.projectControl); - // The /static content, used by docker, development, etc. // This is the stuff that's packaged up via webpack in packages/static. await initStatic(router); @@ -128,8 +126,6 @@ export default async function init(opts: Options): Promise<{ initBlobs(router); initBlobUpload(router); initUpload(router); - initSetCookies(router); - initNatsServer(router); initCustomize(router, opts.isPersonal); initStats(router); initAppRedirect(router); @@ -146,14 +142,32 @@ export default async function init(opts: Options): Promise<{ app, }); + if (opts.conatServer) { + winston.info(`initializing the Conat Server`); + initConatServer({ + httpServer, + ssl: !!opts.cert, + }); + } + if (opts.proxyServer) { - winston.info(`initializing the http proxy server`); + winston.info(`initializing the http proxy server`, { + conatClusterPort, + conatServer: !!opts.conatServer, + isPersonal: opts.isPersonal, + listenersHack: opts.listenersHack, + }); initProxy({ projectControl: opts.projectControl, isPersonal: opts.isPersonal, httpServer, app, listenersHack: opts.listenersHack, + // enable proxy server for /conat if: + // (1) we are not running conat at all from here, or + // (2) we are running socketio in cluster mode, hence + // on a different port + proxyConat: !opts.conatServer || !!conatClusterPort, }); } @@ -164,7 +178,6 @@ export default async function init(opts: Options): Promise<{ // The Next.js server await initNext(app); } - return { httpServer, router }; } diff --git a/src/packages/hub/servers/nats.ts b/src/packages/hub/servers/nats.ts deleted file mode 100644 index 2d551d2113..0000000000 --- a/src/packages/hub/servers/nats.ts +++ /dev/null @@ -1,66 +0,0 @@ -/* -NATS WebSocket proxy -- this primarily just directly proxied the nats -websocket server, so outside browsers can connect to it. - -This assumes there is a NATS server. This gets configured in dev mode -automatically and started via: - -$ cd ~/cocalc/src -$ pnpm nats-server - -*/ - -import { createProxyServer, type ProxyServer } from "http-proxy-3"; -import getLogger from "@cocalc/backend/logger"; -import { type Router } from "express"; -import { natsWebsocketServer } from "@cocalc/backend/data"; -import { - versionCheckFails, - init as initVersionCheck, -} from "@cocalc/hub/proxy/version"; -import { delay } from "awaiting"; - -const logger = getLogger("hub:nats"); - -let proxy: ProxyServer | null = null; -export async function proxyNatsWebsocket(req, socket, head) { - const target = natsWebsocketServer; - logger.debug(`nats proxy -- proxying a connection to ${target}`); - // todo -- allowing no cookie, since that's used by projects and compute servers! - // do NOT disable this until compute servers all set a cookie... which could be a long time. - if (versionCheckFails(req)) { - logger.debug("NATS client failed version check -- closing"); - socket.destroy(); - return; - } - if (proxy == null) { - // make the proxy server - proxy = createProxyServer({ - ws: true, - target, - }); - proxy.on("error", (err) => { - logger.debug(`WARNING: nats websocket proxy error -- ${err}`); - }); - } - - // connect the client's socket to nats via the proxy server: - proxy.ws(req, socket, head); - - while (socket.readyState !== socket.CLOSED) { - if (versionCheckFails(req)) { - logger.debug("NATS client failed version check -- closing"); - setTimeout(() => socket.end(), 10 * 1000); - return; - } - await delay(2 * 60 * 1000); - } -} - -// this is immediately upgraded to a websocket -export function initNatsServer(router: Router) { - initVersionCheck(); - router.get("/nats", async (_req, res) => { - res.send(""); - }); -} diff --git a/src/packages/hub/servers/primus.ts b/src/packages/hub/servers/primus.ts deleted file mode 100644 index 9c37f373a8..0000000000 --- a/src/packages/hub/servers/primus.ts +++ /dev/null @@ -1,61 +0,0 @@ -import { join } from "path"; -import { Router } from "express"; -import Primus from "primus"; -import base_path from "@cocalc/backend/base-path"; -import Logger from "@cocalc/backend/logger"; -import setup_primus_client from "@cocalc/hub/primus-client"; -const { Client } = require("@cocalc/hub/client"); -import { len } from "@cocalc/util/misc"; -import { database } from "./database"; - -interface Options { - httpServer; - router: Router; - projectControl; - clients: { [id: string]: any }; // todo: when client is in typescript, use proper type... - host: string; - port: number; - isPersonal: boolean; -} - -export default function init({ - httpServer, - router, - projectControl, - clients, - host, - port, - isPersonal, -}: Options): void { - const logger = Logger("primus"); - - // It is now safe to change the primusOpts below, and this - // doesn't require changing anything anywhere else. - // See https://github.com/primus/primus#getting-started - const primusOpts = { - pathname: join(base_path, "hub"), - maxLength: 2 * 10485760, // 20MB - twice the default - compression: true, - } as const; - const primus_server = new Primus(httpServer, primusOpts); - logger.info(`listening on ${primusOpts.pathname}`); - - // Make it so new websocket connection requests get handled: - primus_server.on("connection", function (conn) { - // Now handle the connection - logger.info(`new connection from ${conn.address.ip} -- ${conn.id}`); - clients[conn.id] = new Client({ - conn, - logger, - database, - projectControl, - host, - port, - personal: isPersonal, - }); - logger.info(`num_clients=${len(clients)}`); - }); - - // Serve the primus.js client code via the express router. - setup_primus_client(router, primus_server); -} diff --git a/src/packages/hub/servers/version.ts b/src/packages/hub/servers/version.ts deleted file mode 100644 index 80a6aebb22..0000000000 --- a/src/packages/hub/servers/version.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { database } from "./database"; -import { getClients } from "../clients"; -import getServerSettings from "./server-settings"; - -export default async function init() { - if (database.is_standby) { - return; - } - const clients = getClients(); - const settings = await getServerSettings(); - let version_recommended_browser: number = 0; // first time. - const update = () => { - if ( - settings.version.version_recommended_browser == - version_recommended_browser - ) { - // version did not change - return; - } - version_recommended_browser = - settings.version.version_recommended_browser ?? 0; - for (const id in clients) { - const client = clients[id]; - if (client.smc_version < version_recommended_browser) { - client.push_version_update(); - } - } - }; - update(); - settings.table.on("change", update); -} diff --git a/src/packages/hub/tsconfig.json b/src/packages/hub/tsconfig.json index 4c91795874..9d5854ad40 100644 --- a/src/packages/hub/tsconfig.json +++ b/src/packages/hub/tsconfig.json @@ -14,6 +14,6 @@ { "path": "../server" }, { "path": "../static" }, { "path": "../util" }, - { "path": "../nats" } + { "path": "../conat" } ] } diff --git a/src/packages/jupyter/execute/execute-code.ts b/src/packages/jupyter/execute/execute-code.ts index a0f9d79909..c4c15fb334 100644 --- a/src/packages/jupyter/execute/execute-code.ts +++ b/src/packages/jupyter/execute/execute-code.ts @@ -6,8 +6,6 @@ /* Send code to a kernel to be evaluated, then wait for the results and gather them together. - -TODO: for easy testing/debugging, at an "async run() : Messages[]" method. */ import { callback, delay } from "awaiting"; @@ -15,17 +13,21 @@ import { EventEmitter } from "events"; import { VERSION } from "@cocalc/jupyter/kernel/version"; import type { JupyterKernelInterface as JupyterKernel } from "@cocalc/jupyter/types/project-interface"; import type { MessageType } from "@nteract/messaging"; -import { bind_methods, copy_with, deep_copy, uuid } from "@cocalc/util/misc"; -import { +import { copy_with, deep_copy, uuid } from "@cocalc/util/misc"; +import type { CodeExecutionEmitterInterface, + OutputMessage, ExecOpts, StdinFunction, } from "@cocalc/jupyter/types/project-interface"; import { getLogger } from "@cocalc/backend/logger"; +import { EventIterator } from "@cocalc/util/event-iterator"; +import { once } from "@cocalc/util/async-utils"; +import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; const log = getLogger("jupyter:execute-code"); -type State = "init" | "closed" | "running"; +type State = "init" | "running" | "done" | "closed"; export class CodeExecutionEmitter extends EventEmitter @@ -42,12 +44,12 @@ export class CodeExecutionEmitter private iopub_done: boolean = false; private shell_done: boolean = false; private state: State = "init"; - private all_output: object[] = []; private _message: any; private _go_cb: Function | undefined = undefined; private timeout_ms?: number; private timer?: any; private killing: string = ""; + private _iter?: EventIterator; constructor(kernel: JupyterKernel, opts: ExecOpts) { super(); @@ -77,42 +79,81 @@ export class CodeExecutionEmitter allow_stdin: this.stdin != null, }, }; - - bind_methods(this); } - // Emits a valid result - // result is https://jupyter-client.readthedocs.io/en/stable/messaging.html#python-api + // async interface: + iter = (): EventIterator => { + if (this.state == "closed") { + throw Error("closed"); + } + if (this._iter == null) { + this._iter = new EventIterator(this, "output", { + map: (args) => { + if (args[0]?.done) { + setTimeout(() => this._iter?.close(), 1); + } + return args[0]; + }, + }); + } + return this._iter; + }; + + waitUntilDone = reuseInFlight(async () => { + try { + await once(this, "done"); + } catch { + // it throws on close, but that's also "done". + } + }); + + private setState = (state: State) => { + this.state = state; + this.emit(state); + }; + + // Emits a valid result, which is + // https://jupyter-client.readthedocs.io/en/stable/messaging.html#python-api // Or an array of those when this.all is true - emit_output(output: object): void { - this.all_output.push(output); + emit_output = (output: OutputMessage): void => { this.emit("output", output); - } + if (output["done"]) { + this.setState("done"); + } + }; // Call this to inform anybody listening that we've canceled // this execution, and will NOT be doing it ever, and it // was explicitly canceled. - cancel(): void { + cancel = (): void => { this.emit("canceled"); - } + this.setState("done"); + this._iter?.close(); + }; - close(): void { - if (this.state == "closed") return; + close = (): void => { + if (this.state == "closed") { + return; + } + this.setState("closed"); if (this.timer != null) { clearTimeout(this.timer); delete this.timer; } - this.state = "closed"; + this._iter?.close(); + delete this._iter; + // @ts-ignore + delete this._go_cb; this.emit("closed"); this.removeAllListeners(); - } + }; - throw_error(err): void { + throw_error = (err): void => { this.emit("error", err); this.close(); - } + }; - async _handle_stdin(mesg: any): Promise { + private _handle_stdin = async (mesg: any): Promise => { if (!this.stdin) { throw Error("BUG -- stdin handling not supported"); } @@ -154,9 +195,9 @@ export class CodeExecutionEmitter }; log.silly("_handle_stdin: STDIN server --> kernel:", m); this.kernel.channel?.next(m); - } + }; - _handle_shell(mesg: any): void { + private _handle_shell = (mesg: any): void => { if (mesg.parent_header.msg_id !== this._message.header.msg_id) { log.silly( `_handle_shell: msg_id mismatch: ${mesg.parent_header.msg_id} != ${this._message.header.msg_id}`, @@ -182,23 +223,23 @@ export class CodeExecutionEmitter } this.set_shell_done(true); } - } + }; - private set_shell_done(value: boolean): void { + private set_shell_done = (value: boolean): void => { this.shell_done = value; if (this.iopub_done && this.shell_done) { this._finish(); } - } + }; - private set_iopub_done(value: boolean): void { + private set_iopub_done = (value: boolean): void => { this.iopub_done = value; if (this.iopub_done && this.shell_done) { this._finish(); } - } + }; - _handle_iopub(mesg: any): void { + _handle_iopub = (mesg: any): void => { if (mesg.parent_header.msg_id !== this._message.header.msg_id) { // iopub message for a different execute request so ignore it. return; @@ -219,16 +260,16 @@ export class CodeExecutionEmitter this.set_iopub_done( !!this.killing || mesg.content?.execution_state == "idle", ); - } + }; // Called if the kernel is closed for some reason, e.g., crashing. - private handle_closed(): void { - log.debug("CodeExecutionEmitter.handle_closed: kernel closed"); + private handleClosed = (): void => { + log.debug("CodeExecutionEmitter.handleClosed: kernel closed"); this.killing = "kernel crashed"; this._finish(); - } + }; - _finish(): void { + private _finish = (): void => { if (this.state == "closed") { return; } @@ -241,7 +282,8 @@ export class CodeExecutionEmitter this.kernel._execute_code_queue.shift(); // finished this.kernel._process_execute_code_queue(); // start next exec } - this.kernel.removeListener("close", this.handle_closed); + this.kernel.removeListener("closed", this.handleClosed); + this.kernel.removeListener("failed", this.handleClosed); this._push_mesg({ done: true }); this.close(); @@ -253,9 +295,9 @@ export class CodeExecutionEmitter // not have randomly done so itself in output. this._go_cb?.(this.killing); this._go_cb = undefined; - } + }; - _push_mesg(mesg): void { + _push_mesg = (mesg): void => { // TODO: mesg isn't a normal javascript object; // it's **silently** immutable, which // is pretty annoying for our use. For now, we @@ -267,24 +309,25 @@ export class CodeExecutionEmitter mesg.msg_type = header.msg_type; } this.emit_output(mesg); - } + }; - async go(): Promise { + go = async (): Promise => { await callback(this._go); - return this.all_output; - } + }; - _go(cb: Function): void { + private _go = (cb: Function): void => { if (this.state != "init") { cb("may only run once"); return; } this.state = "running"; log.silly("_execute_code", this.code); - if (this.kernel.get_state() === "closed") { - log.silly("_execute_code", "kernel.get_state() is closed"); - this.close(); - cb("closed - jupyter - execute_code"); + const kernelState = this.kernel.get_state(); + if (kernelState == "closed" || kernelState == "failed") { + log.silly("_execute_code", "kernel.get_state() is ", kernelState); + this.killing = kernelState; + this._finish(); + cb(kernelState); return; } @@ -296,18 +339,22 @@ export class CodeExecutionEmitter this.kernel.on("shell", this._handle_shell); this.kernel.on("iopub", this._handle_iopub); - log.debug("_execute_code: send the message to get things rolling"); - this.kernel.channel?.next(this._message); - - this.kernel.on("closed", this.handle_closed); + this.kernel.once("closed", this.handleClosed); + this.kernel.once("failed", this.handleClosed); if (this.timeout_ms) { // setup a timeout at which point things will get killed if they don't finish this.timer = setTimeout(this.timeout, this.timeout_ms); } - } - private async timeout(): Promise { + log.debug("_execute_code: send the message to get things rolling"); + if (this.kernel.channel == null) { + throw Error("bug -- channel must be defined"); + } + this.kernel.channel.next(this._message); + }; + + private timeout = async (): Promise => { if (this.state == "closed") { log.debug( "CodeExecutionEmitter.timeout: already finished, so nothing to worry about", @@ -340,5 +387,5 @@ export class CodeExecutionEmitter this.kernel.signal("SIGKILL"); this._finish(); } - } + }; } diff --git a/src/packages/jupyter/kernel/nats-service.ts b/src/packages/jupyter/kernel/conat-service.ts similarity index 92% rename from src/packages/jupyter/kernel/nats-service.ts rename to src/packages/jupyter/kernel/conat-service.ts index cac33913e7..26655a3274 100644 --- a/src/packages/jupyter/kernel/nats-service.ts +++ b/src/packages/jupyter/kernel/conat-service.ts @@ -3,17 +3,17 @@ DEVELOPMENT: -Go to packages/project/nats/open-files.ts and for a dev project, stop the built in open files service +Go to packages/project/c/open-files.ts and for a dev project, stop the built in open files service and start your own in a terminal. If you then open a jupyter notebook in that project, you can use your terminal running the open files service to interact with anything here from the server size. In particular, set global.x = ..., etc. */ -import { createNatsJupyterService } from "@cocalc/nats/service/jupyter"; +import { createConatJupyterService } from "@cocalc/conat/service/jupyter"; import { get_existing_kernel as getKernel } from "@cocalc/jupyter/kernel"; import { bufferToBase64 } from "@cocalc/util/base64"; -export async function initNatsService({ +export async function initConatService({ path, project_id, }: { @@ -108,7 +108,7 @@ export async function initNatsService({ return { buffer64: bufferToBase64(buffer) }; }, }; - return await createNatsJupyterService({ + return await createConatJupyterService({ project_id, path, impl, diff --git a/src/packages/jupyter/kernel/get-ports.ts b/src/packages/jupyter/kernel/get-ports.ts deleted file mode 100644 index ef41891817..0000000000 --- a/src/packages/jupyter/kernel/get-ports.ts +++ /dev/null @@ -1,45 +0,0 @@ -/* - -Return n available ports, with options as specified in the portfinder module -(see https://github.com/http-party/node-portfinder#readme). - -The difference between this and portfinder is that no matter what, -this function will never return a port it has aleady within 1 minute. -This avoids a major race condition, e.g., when creating multiple -jupyter notebooks at nearly the same time. -*/ - -import { promisify } from "node:util"; -import { getPorts as getPorts0 } from "portfinder"; -import LRU from "lru-cache"; - -const getPortsUnsafe = promisify(getPorts0 as any); - -const cache = new LRU({ - ttl: 60000, - max: 10000, -}); - -export default async function getPorts( - n: number, - options: { - port?: number; // minimum port - stopPort?: number; // maximum port - } = {} -): Promise { - const ports: number[] = []; - while (ports.length < n) { - for (const port of await getPortsUnsafe(n - ports.length, options)) { - if (!cache.has(port)) { - cache.set(port, true); - ports.push(port); - } - } - if (ports.length < n) { - // we have to change something, otherwise getPortsUnsafe will never - // give us anything useful and we're stuck in a loop. - options = { ...options, port: (options.port ?? 8000) + 1 }; - } - } - return ports; -} diff --git a/src/packages/jupyter/kernel/kernel-data.test.ts b/src/packages/jupyter/kernel/kernel-data.test.ts new file mode 100644 index 0000000000..a40d87b22c --- /dev/null +++ b/src/packages/jupyter/kernel/kernel-data.test.ts @@ -0,0 +1,31 @@ +import { + get_kernel_data, + get_kernel_data_by_name, + getLanguage, + getPythonKernelName, +} from "@cocalc/jupyter/kernel/kernel-data"; + +describe("basic consistency checks with getting kernels", () => { + let kernels; + it("gets the kernels", async () => { + kernels = await get_kernel_data(); + expect(kernels.length).toBeGreaterThan(0); + }); + + it("for each kernel above, call get_kernel_data_by_name", async () => { + for (const x of kernels) { + const d = await get_kernel_data_by_name(x.name); + expect(d).toEqual(x); + } + }); + + it("for each kernel above, call getLanguage", async () => { + for (const x of kernels) { + await getLanguage(x.name); + } + }); + + it("get a python kernel", async () => { + await getPythonKernelName(); + }); +}); diff --git a/src/packages/jupyter/kernel/kernel-data.ts b/src/packages/jupyter/kernel/kernel-data.ts index f9ed5dbb42..3989580526 100644 --- a/src/packages/jupyter/kernel/kernel-data.ts +++ b/src/packages/jupyter/kernel/kernel-data.ts @@ -14,7 +14,7 @@ Specs: https://jupyter-client.readthedocs.io/en/stable/kernels.html#kernel-specs This is supposed to be basically the same as "jupyter kernelspec list --json", but it is NOT always. E.g., on my dev system the "ptyhon3" system-wide kernel is just completely missed. Also, "jupyter kernelspec list --json" is MUCH slower, taking almost a second, versus only -a few ms for this. We stick with this for now, but may need to improve upstream. +a few ms for this. We stick with this for now, but may need to improve. */ import { findAll } from "kernelspecs"; @@ -106,3 +106,15 @@ export async function get_kernel_data_by_name( } throw Error(`no such kernel '${name}'`); } + +// return the name of a python kernel -- very useful for unit testing. +export async function getPythonKernelName() { + const kernels = await get_kernel_data(); + for (const x of kernels) { + const name = x.name.toLowerCase(); + if (name.includes("python") && !name.includes("python2")) { + return x.name; + } + } + throw Error("no python kernels"); +} diff --git a/src/packages/jupyter/kernel/kernel-execute-code.test.ts b/src/packages/jupyter/kernel/kernel-execute-code.test.ts new file mode 100644 index 0000000000..1dce8b68b7 --- /dev/null +++ b/src/packages/jupyter/kernel/kernel-execute-code.test.ts @@ -0,0 +1,75 @@ +/* +I'm a little hesistant about testing this since we'll need to make sure that a kernel is +installed, e.g., to test on Github actions. +Probably, the way to go would be to install https://www.npmjs.com/package/ijavascript +and just test that a lot, since it would be the minimal dependency. + +There are a lot of ideas for tests in this bitrotted place: + +https://github.com/sagemathinc/cocalc/tree/master/src/packages/project/jupyter/test +*/ + +import expect from "expect"; +import { getPythonKernel, closeKernels } from "./kernel.test"; + + +describe("a kernel implicitly spawns when you execute code", () => { + let k; + it("get and do NOT spawn a python kernel", async () => { + k = await getPythonKernel("python-spawn.ipynb"); + }); + + it.skip("start some code running and see spawning is automatic", async () => { + const code = "import os; os.getpid()"; + const output = k.execute_code({ code }); + for await (const out of output.iter()) { + if (out.content?.code) { + expect(out.content.code).toBe(code); + } + if (out.content?.data) { + const pid = out.content?.data["text/plain"]; + expect(parseInt(pid)).toEqual(k.pid()); + break; + } + } + }); + + it("cleans up", () => { + k.close(); + }); +}); + +describe("test execute_code_now and chdir", () => { + let k; + it("get a python kernel", async () => { + k = await getPythonKernel("python-chdir.ipynb"); + }); + + it("also test the execute_code_now method", async () => { + const out = await k.execute_code_now({ code: "2+3" }); + const v = out.filter((x) => x.content?.data); + expect(v[0].content.data["text/plain"]).toBe("5"); + }); + + it("also test the chdir method", async () => { + // before + const out = await k.execute_code_now({ code: "import os; os.curdir" }); + const v = out.filter((x) => x.content?.data); + expect(v[0].content.data["text/plain"]).toBe("'.'"); + + await k.chdir("/tmp"); + const out2 = await k.execute_code_now({ + code: "os.path.abspath(os.curdir)", + }); + const v2 = out2.filter((x) => x.content?.data); + expect(v2[0].content.data["text/plain"]).toBe("'/tmp'"); + }); + + it("cleans up", () => { + k.close(); + }); +}); + +afterAll(() => { + closeKernels(); +}); diff --git a/src/packages/jupyter/kernel/kernel.test.ts b/src/packages/jupyter/kernel/kernel.test.ts index 10d849acf5..a9f9e0c8fe 100644 --- a/src/packages/jupyter/kernel/kernel.test.ts +++ b/src/packages/jupyter/kernel/kernel.test.ts @@ -10,13 +10,141 @@ https://github.com/sagemathinc/cocalc/tree/master/src/packages/project/jupyter/t */ import expect from "expect"; -import { kernel } from "./kernel"; +import { kernel, type JupyterKernel } from "./kernel"; +import { getPythonKernelName } from "./kernel-data"; + +const usedNames = new Set(); +const kernels: JupyterKernel[] = []; +export async function getPythonKernel( + path: string, + noCheck = false, +): Promise { + if (!noCheck && usedNames.has(path)) { + throw Error(`do not reuse names as that is very confusing -- ${path}`); + } + usedNames.add(path); + const k = kernel({ name: await getPythonKernelName(), path }); + kernels.push(k); + return k; +} + +export function closeKernels() { + kernels.map((k) => k.close()); +} describe("test trying to use a kernel that doesn't exist", () => { - it("fails", async () => { - const k = kernel({ name: "no-such-kernel", path: "x.ipynb" }); - await expect(k.execute_code_now({ code: "2+3" })).rejects.toThrow( - "No spec available for kernel" - ); + it("fails to start", async () => { + const k = kernel({ name: "no-such-kernel", path: "none.ipynb" }); + const errors: any[] = []; + k.on("kernel_error", (err) => { + errors.push(err); + }); + + await expect( + async () => await k.execute_code_now({ code: "2+3" }), + ).rejects.toThrow("No spec available for kernel"); + + expect(errors[0]).toContain("No spec available for kernel"); + }); +}); + +describe("create and close python kernel", () => { + let k; + it("get a python kernel", async () => { + k = await getPythonKernel("python-0.ipynb"); + }); + + it("cleans up", () => { + k.close(); + }); +}); + +describe("spawn and close python kernel", () => { + let k; + it("get a python kernel", async () => { + k = await getPythonKernel("python-1.ipynb"); + }); + + it("spawns the kernel", async () => { + await k.spawn(); + }); + + it("cleans up", () => { + k.close(); + }); +}); + +describe("compute 2+3 using a python kernel", () => { + let k; + it("get a python kernel", async () => { + k = await getPythonKernel("python-2.ipynb"); + }); + + it("spawn the kernel", async () => { + await k.spawn(); }); + + it("evaluate 2+3, confirming the result", async () => { + const output = k.execute_code({ code: "2+3" }); + const iter = output.iter(); + const v: any[] = []; + for await (const x of iter) { + v.push(x); + } + expect(v[0].content).toEqual({ execution_state: "busy" }); + expect(v[1].content).toEqual({ code: "2+3", execution_count: 1 }); + expect(v[2].content.data).toEqual({ "text/plain": "5" }); + }); + + it("define a variable in one call, then use it in another", async () => { + const output = k.execute_code({ code: "a=5" }); + await output.waitUntilDone(); + output.close(); + }); + + it("uses that variable in another call", async () => { + const output = k.execute_code({ code: "a + a" }); + const iter = output.iter(); + await output.waitUntilDone(); + for await (const x of iter) { + if (x.content?.data) { + expect(x.content?.data).toEqual({ "text/plain": "10" }); + break; + } + } + }); + + it("cleans up", () => { + k.close(); + }); +}); + + +describe("start computation then immediately close the kernel should not crash", () => { + let k; + it("get and spawn a python kernel", async () => { + k = await getPythonKernel("python-4.ipynb"); + await k.spawn(); + }); + + it("start something running, then immediately close and see error event is called", async () => { + const output = k.execute_code({ code: "sleep 10000" }); + for await (const _ of output.iter()) { + // it's ack'd as running: + break; + } + }); + + it("closes during computation", () => { + k.close(); + }); + + it("starts and closes another kernel with the same path", async () => { + const k2 = await getPythonKernel("python-4.ipynb", true); + k2.close(); + }); +}); + +afterAll(() => { + closeKernels(); }); diff --git a/src/packages/jupyter/kernel/kernel.ts b/src/packages/jupyter/kernel/kernel.ts index c676900fe8..43198460d8 100644 --- a/src/packages/jupyter/kernel/kernel.ts +++ b/src/packages/jupyter/kernel/kernel.ts @@ -15,6 +15,9 @@ $ node */ +// POOL VERSION - faster to restart but possible subtle issues +const USE_KERNEL_POOL = true; + // const DEBUG = true; // only for extreme debugging. const DEBUG = false; // normal mode if (DEBUG) { @@ -28,8 +31,11 @@ if (DEBUG) { import nodeCleanup from "node-cleanup"; import type { Channels, MessageType } from "@nteract/messaging"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; -import { callback, delay } from "awaiting"; -import { createMainChannel } from "enchannel-zmq-backend"; +import { callback } from "awaiting"; +import { + createMainChannel, + closeSockets, +} from "@cocalc/jupyter/zmq/jupyter-channels"; import { EventEmitter } from "node:events"; import { unlink } from "@cocalc/backend/misc/async-utils-node"; import { remove_redundant_reps } from "@cocalc/jupyter/ipynb/import-from-ipynb"; @@ -44,7 +50,7 @@ import { import { JupyterStore } from "@cocalc/jupyter/redux/store"; import { JUPYTER_MIMETYPES } from "@cocalc/jupyter/util/misc"; import type { SyncDB } from "@cocalc/sync/editor/db/sync"; -import { retry_until_success } from "@cocalc/util/async-utils"; +import { retry_until_success, until } from "@cocalc/util/async-utils"; import createChdirCommand from "@cocalc/util/jupyter-api/chdir-commands"; import { key_value_store } from "@cocalc/util/key-value-store"; import { @@ -63,11 +69,15 @@ import { getLanguage, get_kernel_data_by_name, } from "@cocalc/jupyter/kernel/kernel-data"; + import launchJupyterKernel, { LaunchJupyterOpts, SpawnedKernel, killKernel, } from "@cocalc/jupyter/pool/pool"; +// non-pool version +import launchJupyterKernelNoPool from "@cocalc/jupyter/kernel/launch-kernel"; + import { getAbsolutePathFromHome } from "@cocalc/jupyter/util/fs"; import type { KernelParams } from "@cocalc/jupyter/types/kernel"; import { redux_name } from "@cocalc/util/redux/name"; @@ -220,13 +230,18 @@ nodeCleanup(() => { // NOTE: keep JupyterKernel implementation private -- use the kernel function // above, and the interface defined in types. -class JupyterKernel extends EventEmitter implements JupyterKernelInterface { +export class JupyterKernel + extends EventEmitter + implements JupyterKernelInterface +{ // name -- if undefined that means "no actual Jupyter kernel" (i.e., this JupyterKernel exists // here, but there is no actual separate real Jupyter kernel process and one won't be created). // Everything should work, except you can't *spawn* such a kernel. public name: string | undefined; - public store: any; // this is a key:value store used mainly for stdin support right now. NOTHING TO DO WITH REDUX! + // this is a key:value store used mainly for stdin support right now. NOTHING TO DO WITH REDUX! + public store: any; + public readonly identity: string = uuid(); private stderr: string = ""; @@ -241,6 +256,7 @@ class JupyterKernel extends EventEmitter implements JupyterKernelInterface { public _execute_code_queue: CodeExecutionEmitter[] = []; public channel?: Channels; private has_ensured_running: boolean = false; + private failedError: string = ""; constructor( name: string | undefined, @@ -260,7 +276,7 @@ class JupyterKernel extends EventEmitter implements JupyterKernelInterface { const { head, tail } = path_split(getAbsolutePathFromHome(this._path)); this._directory = head; this._filename = tail; - this._set_state("off"); + this.setState("off"); this._execute_code_queue = []; if (_jupyter_kernels[this._path] !== undefined) { // This happens when we change the kernel for a given file, e.g., from python2 to python3. @@ -278,7 +294,7 @@ class JupyterKernel extends EventEmitter implements JupyterKernelInterface { }; // no-op if calling it doesn't change the state. - private _set_state = (state: State): void => { + private setState = (state: State): void => { // state = 'off' --> 'spawning' --> 'starting' --> 'running' --> 'closed' // 'failed' if (this._state == state) return; @@ -287,96 +303,104 @@ class JupyterKernel extends EventEmitter implements JupyterKernelInterface { this.emit(this._state); // we *SHOULD* use this everywhere, not above. }; + private setFailed = (error: string): void => { + this.failedError = error; + this.emit("kernel_error", error); + this.setState("failed"); + }; + get_state = (): string => { return this._state; }; - spawn = reuseInFlight( - async (spawn_opts?: { env?: { [key: string]: string } }): Promise => { - if (this._state === "closed") { - // game over! - throw Error("closed -- kernel spawn"); - } - if (!this.name) { - // spawning not allowed. - throw Error("cannot spawn since no kernel is set"); - } - if (["running", "starting"].includes(this._state)) { - // Already spawned, so no need to do it again. - return; - } - this._set_state("spawning"); - const dbg = this.dbg("spawn"); - dbg("spawning kernel..."); - - // **** - // CRITICAL: anything added to opts better not be specific - // to the kernel path or it will completely break using a - // pool, which makes things massively slower. - // **** - - const opts: LaunchJupyterOpts = { - env: spawn_opts?.env ?? {}, - ...(this.ulimit != null ? { ulimit: this.ulimit } : undefined), - }; + private spawnedAlready = false; + spawn = async (spawn_opts?: { + env?: { [key: string]: string }; + }): Promise => { + if (this._state === "closed") { + // game over! + throw Error("closed -- kernel spawn"); + } + if (!this.name) { + // spawning not allowed. + throw Error("cannot spawn since no kernel is set"); + } + if (["running", "starting"].includes(this._state)) { + // Already spawned, so no need to do it again. + return; + } - try { - const kernelData = await get_kernel_data_by_name(this.name); - // This matches "sage", "sage-x.y", and Sage Python3 ("sage -python -m ipykernel") - if (kernelData.argv[0].startsWith("sage")) { - dbg("setting special environment for Sage kernels"); - opts.env = merge(opts.env, SAGE_JUPYTER_ENV); - } - } catch (err) { - dbg(`No kernelData available for ${this.name}`); - } + if (this.spawnedAlready) { + return; + } + this.spawnedAlready = true; + + this.setState("spawning"); + const dbg = this.dbg("spawn"); + dbg("spawning kernel..."); - // Make cocalc default to the colab renderer for cocalc-jupyter, since - // this one happens to work best for us, and they don't have a custom - // one for us. See https://plot.ly/python/renderers/ and - // https://github.com/sagemathinc/cocalc/issues/4259 - opts.env.PLOTLY_RENDERER = "colab"; - opts.env.COCALC_JUPYTER_KERNELNAME = this.name; - - // !!! WARNING: do NOT add anything new here that depends on that path!!!! - // Otherwise the pool will switch to falling back to not being used, and - // cocalc would then be massively slower. - // Non-uniform customization. - // launchJupyterKernel is explicitly smart enough to deal with opts.cwd - if (this._directory) { - opts.cwd = this._directory; + // **** + // CRITICAL: anything added to opts better not be specific + // to the kernel path or it will completely break using a + // pool, which makes things massively slower. + // **** + + const opts: LaunchJupyterOpts = { + env: spawn_opts?.env ?? {}, + ulimit: this.ulimit, + }; + + try { + const kernelData = await get_kernel_data_by_name(this.name); + // This matches "sage", "sage-x.y", and Sage Python3 ("sage -python -m ipykernel") + if (kernelData.argv[0].startsWith("sage")) { + dbg("setting special environment for Sage kernels"); + opts.env = merge(opts.env, SAGE_JUPYTER_ENV); } - // launchJupyterKernel is explicitly smart enough to deal with opts.env.COCALC_JUPYTER_FILENAME - opts.env.COCALC_JUPYTER_FILENAME = this._path; - // and launchJupyterKernel is NOT smart enough to deal with anything else! + } catch (err) { + dbg(`No kernelData available for ${this.name}`); + } - try { - dbg("launching kernel interface..."); + // Make cocalc default to the colab renderer for cocalc-jupyter, since + // this one happens to work best for us, and they don't have a custom + // one for us. See https://plot.ly/python/renderers/ and + // https://github.com/sagemathinc/cocalc/issues/4259 + opts.env.PLOTLY_RENDERER = "colab"; + opts.env.COCALC_JUPYTER_KERNELNAME = this.name; + + // !!! WARNING: do NOT add anything new here that depends on that path!!!! + // Otherwise the pool will switch to falling back to not being used, and + // cocalc would then be massively slower. + // Non-uniform customization. + // launchJupyterKernel is explicitly smart enough to deal with opts.cwd + if (this._directory) { + opts.cwd = this._directory; + } + // launchJupyterKernel is explicitly smart enough to deal with opts.env.COCALC_JUPYTER_FILENAME + opts.env.COCALC_JUPYTER_FILENAME = this._path; + // and launchJupyterKernel is NOT smart enough to deal with anything else! + + try { + if (USE_KERNEL_POOL) { + dbg("launching Jupyter kernel, possibly from pool"); this._kernel = await launchJupyterKernel(this.name, opts); - await this.finish_spawn(); - } catch (err) { - dbg("ERROR spawning kernel", err); - // @ts-ignore - if (this._state == "closed") { - throw Error("closed -- kernel spawn later"); - } - this._set_state("failed"); - this.emit( - "kernel_error", - `**Unable to Spawn Jupyter Kernel:**\n\n${err} \n\nTry this in a terminal to help debug this (or contact support): \`jupyter console --kernel=${this.name}\`\n\nOnce you fix the problem, explicitly restart this kernel to test here.`, - ); - throw err; + } else { + dbg("launching Jupyter kernel, NOT using pool"); + this._kernel = await launchJupyterKernelNoPool(this.name, opts); } - - // NOW we do path-related customizations: - // TODO: we will set each of these after getting a kernel from the pool - // expose path of jupyter notebook -- https://github.com/sagemathinc/cocalc/issues/5165 - //opts.env.COCALC_JUPYTER_FILENAME = this._path; - // if (this._directory !== "") { - // opts.cwd = this._directory; - // } - }, - ); + dbg("finishing kernel setup"); + await this.finishSpawningKernel(); + } catch (err) { + dbg(`ERROR spawning kernel - ${err}, ${err.stack}`); + // @ts-ignore + if (this._state == "closed") { + throw Error("closed"); + } + this.setFailed( + `**Unable to Spawn Jupyter Kernel:**\n\n${err} \n\nTry this in a terminal to help debug this (or contact support): \`jupyter console --kernel=${this.name}\`\n\nOnce you fix the problem, explicitly restart this kernel to test here.`, + ); + } + }; get_spawned_kernel = () => { return this._kernel; @@ -386,8 +410,8 @@ class JupyterKernel extends EventEmitter implements JupyterKernelInterface { return this._kernel?.connectionFile; }; - private finish_spawn = async () => { - const dbg = this.dbg("finish_spawn"); + private finishSpawningKernel = async () => { + const dbg = this.dbg("finishSpawningKernel"); dbg("now finishing spawn of kernel..."); if (DEBUG) { @@ -400,8 +424,7 @@ class JupyterKernel extends EventEmitter implements JupyterKernelInterface { this._kernel.spawn.on("error", (err) => { const error = `${err}\n${this.stderr}`; dbg("kernel error", error); - this.emit("kernel_error", error); - this._set_state("off"); + this.setFailed(error); }); // Track stderr from the subprocess itself (the kernel). @@ -421,10 +444,9 @@ class JupyterKernel extends EventEmitter implements JupyterKernelInterface { this._kernel.spawn.stdout.on("data", (_data) => { // NOTE: it is very important to read stdout (and stderr above) - // even if we **totally ignore** the data. Otherwise, execa saves - // some amount then just locks up and doesn't allow flushing the - // output stream. This is a "nice" feature of execa, since it means - // no data gets dropped. See https://github.com/sagemathinc/cocalc/issues/5065 + // even if we **totally ignore** the data. Otherwise, exec + // might overflow + // https://github.com/sagemathinc/cocalc/issues/5065 }); dbg("create main channel...", this._kernel.config); @@ -434,54 +456,68 @@ class JupyterKernel extends EventEmitter implements JupyterKernelInterface { // Thus we do some tests, waiting for at least 2 seconds for there // to be a pid. This is complicated and ugly, and I'm sorry about that, // but sometimes that's life. - let i = 0; - while (i < 20 && this._state == "spawning" && !this._kernel?.spawn?.pid) { - i += 1; - await delay(100); - } - if (this._state != "spawning" || !this._kernel?.spawn?.pid) { - if (this._state == "spawning") { - this.emit("kernel_error", "Failed to start kernel process."); - this._set_state("off"); - } + try { + await until( + () => { + if (this._state != "spawning") { + // gave up + return true; + } + if (this.pid()) { + // there's a process :-) + return true; + } + return false; + }, + { start: 100, max: 100, timeout: 3000 }, + ); + } catch (err) { + // timed out + this.setFailed(`Failed to start kernel process. ${err}`); return; } - const local = { success: false, gaveUp: false }; + if (this._state != "spawning") { + // got canceled + return; + } + const pid = this.pid(); + if (!pid) { + throw Error("bug"); + } + let success = false; + let gaveUp = false; setTimeout(() => { - if (!local.success) { - local.gaveUp = true; + if (!success) { + gaveUp = true; // it's been 30s and the channels didn't work. Let's give up. // probably the kernel process just failed. - this.emit("kernel_error", "Failed to start kernel process."); - this._set_state("off"); - // We can't "cancel" createMainChannel itself -- that will require + this.setFailed("Failed to start kernel process -- timeout"); + // We can't yet "cancel" createMainChannel itself -- that will require // rewriting that dependency. // https://github.com/sagemathinc/cocalc/issues/7040 + // I did rewrite that -- so let's revisit this! } }, MAX_KERNEL_SPAWN_TIME); - const channel = await createMainChannel( - this._kernel.config, - "", - this.identity, - ); - if (local.gaveUp) { + const channel = await createMainChannel(this._kernel.config, this.identity); + if (gaveUp) { + process.kill(-pid, 9); return; } this.channel = channel; - local.success = true; + success = true; dbg("created main channel"); this.channel?.subscribe((mesg) => { switch (mesg.channel) { case "shell": - this._set_state("running"); + this.setState("running"); this.emit("shell", mesg); break; case "stdin": this.emit("stdin", mesg); break; case "iopub": - this._set_state("running"); + this.setState("running"); if (mesg.content != null && mesg.content.execution_state != null) { this.emit("execution_state", mesg.content.execution_state); } @@ -502,7 +538,7 @@ class JupyterKernel extends EventEmitter implements JupyterKernelInterface { } }); - this._kernel.spawn.on("exit", (exit_code, signal) => { + this._kernel.spawn.once("exit", (exit_code, signal) => { if (this._state === "closed") { return; } @@ -511,31 +547,30 @@ class JupyterKernel extends EventEmitter implements JupyterKernelInterface { ); const stderr = this.stderr ? `\n...\n${this.stderr}` : ""; if (signal != null) { - this.emit( - "kernel_error", - `Kernel last terminated by signal ${signal}.${stderr}`, - ); + this.setFailed(`Kernel last terminated by signal ${signal}.${stderr}`); } else if (exit_code != null) { - this.emit( - "kernel_error", - `Kernel last exited with code ${exit_code}.${stderr}`, - ); + this.setFailed(`Kernel last exited with code ${exit_code}.${stderr}`); } this.close(); }); - // so we can start sending code execution to the kernel, etc. - this._set_state("starting"); + if (this._state == "spawning") { + // so we can start sending code execution to the kernel, etc. + this.setState("starting"); + } + }; + + pid = (): number | undefined => { + return this._kernel?.spawn?.pid; }; // Signal should be a string like "SIGINT", "SIGKILL". // See https://nodejs.org/api/process.html#process_process_kill_pid_signal signal = (signal: string): void => { const dbg = this.dbg("signal"); - const spawn = this._kernel != null ? this._kernel.spawn : undefined; - const pid = spawn?.pid; + const pid = this.pid(); dbg(`pid=${pid}, signal=${signal}`); - if (pid == null) { + if (!pid) { return; } try { @@ -546,15 +581,13 @@ class JupyterKernel extends EventEmitter implements JupyterKernelInterface { } }; - // This is async, but the process.kill happens *before* - // anything async. That's important for cleaning these - // up when the project terminates. - close = async (): Promise => { + close = (): void => { this.dbg("close")(); if (this._state === "closed") { return; } - this._set_state("closed"); + closeSockets(this.identity); + this.setState("closed"); if (this.store != null) { this.store.close(); delete this.store; @@ -570,8 +603,8 @@ class JupyterKernel extends EventEmitter implements JupyterKernelInterface { delete this.channel; } if (this._execute_code_queue != null) { - for (const code_snippet of this._execute_code_queue) { - code_snippet.close(); + for (const runningCode of this._execute_code_queue) { + runningCode.close(); } this._execute_code_queue = []; } @@ -615,10 +648,13 @@ class JupyterKernel extends EventEmitter implements JupyterKernelInterface { } dbg("spawning"); await this.spawn(); + if (this.get_state() != "starting" && this.get_state() != "running") { + return; + } if (this._kernel?.initCode != null) { for (const code of this._kernel?.initCode ?? []) { dbg("initCode ", code); - await new CodeExecutionEmitter(this, { code }).go(); + this.execute_code({ code }, true); } } if (!this.has_ensured_running) { @@ -705,9 +741,9 @@ class JupyterKernel extends EventEmitter implements JupyterKernelInterface { ); try { await this.ensure_running(); - this._execute_code_queue[0].go(); + await this._execute_code_queue[0].go(); } catch (err) { - dbg(`error running kernel -- ${err}`); + dbg(`WARNING: error running kernel -- ${err}`); for (const code of this._execute_code_queue) { code.throw_error(err); } @@ -736,20 +772,27 @@ class JupyterKernel extends EventEmitter implements JupyterKernelInterface { this._execute_code_queue = []; }; - // This is like execute_code, but async and returns all the results, - // and does not use the internal execution queue. - // This is used for unit testing and interactive work at the terminal and nbgrader and the stateless api. + // This is like execute_code, but async and returns all the results. + // This is used for unit testing and interactive work at + // the terminal and nbgrader and the stateless api. execute_code_now = async (opts: ExecOpts): Promise => { this.dbg("execute_code_now")(); - if (this._state === "closed") { - throw Error("closed -- kernel -- execute_code_now"); + if (this._state == "closed") { + throw Error("closed"); } - if (opts.halt_on_error === undefined) { - // if not specified, default to true. - opts.halt_on_error = true; + if (this.failedError) { + throw Error(this.failedError); + } + const output = this.execute_code({ halt_on_error: true, ...opts }); + const v: object[] = []; + for await (const mesg of output.iter()) { + v.push(mesg); + } + if (this.failedError) { + // kernel failed during call + throw Error(this.failedError); } - await this.ensure_running(); - return await new CodeExecutionEmitter(this, opts).go(); + return v; }; private saveBlob = (data: string, type: string) => { @@ -943,7 +986,6 @@ class JupyterKernel extends EventEmitter implements JupyterKernelInterface { }, ); - // TODO: double check that this actually returns sha1 load_attachment = async (path: string): Promise => { const dbg = this.dbg("load_attachment"); dbg(`path='${path}'`); @@ -1107,8 +1149,8 @@ class JupyterKernel extends EventEmitter implements JupyterKernelInterface { const absPath = getAbsolutePathFromHome(path); const code = createChdirCommand(lang, absPath); + // code = '' if no command needed, e.g., for sparql. if (code) { - // returns '' if no command needed, e.g., for sparql. await this.execute_code_now({ code }); } }; diff --git a/src/packages/jupyter/kernel/launch-kernel.ts b/src/packages/jupyter/kernel/launch-kernel.ts index 9ccd519e92..dec3e8ec41 100644 --- a/src/packages/jupyter/kernel/launch-kernel.ts +++ b/src/packages/jupyter/kernel/launch-kernel.ts @@ -1,5 +1,5 @@ // This file allows you to run a jupyter kernel via `launch_jupyter_kernel`. -// You have to provide the kernel name and (optionally) launch options for execa [1]. +// You have to provide the kernel name and (optionally) launch options. // // Example: // import launchJupyterKernel from "./launch-jupyter-kernel"; @@ -9,9 +9,6 @@ // * `kernel.spawn` holds the process and you have to close it when finished. // * Unless `cleanupConnectionFile` is false, the connection file will be deleted when finished. // -// Ref: -// [1] execa: https://github.com/sindresorhus/execa#readme -// // History: // This is a port of https://github.com/nteract/spawnteract/ to TypeScript (with minor changes). // Original license: BSD-3-Clause and this file is also licensed under BSD-3-Clause! @@ -23,24 +20,18 @@ import * as fs from "fs"; import * as uuid from "uuid"; import { mkdir } from "fs/promises"; import { spawn } from "node:child_process"; - import { findAll } from "kernelspecs"; import * as jupyter_paths from "jupyter-paths"; - -import getPorts from "./get-ports"; +import bash from "@cocalc/backend/bash"; import { writeFile } from "jsonfile"; import mkdirp from "mkdirp"; import shellEscape from "shell-escape"; import { envForSpawn } from "@cocalc/backend/misc"; import { getLogger } from "@cocalc/backend/logger"; +import { getPorts } from "@cocalc/backend/get-port"; const logger = getLogger("launch-kernel"); -// This is temporary hack to import the latest execa, which is only -// available as an ES Module now. We will of course eventually switch -// to using esm modules instead of commonjs, but that's a big project. -import { dynamicImport } from "tsimportlib"; - // this is passed to "execa", there are more options // https://github.com/sindresorhus/execa#options // https://nodejs.org/dist/latest-v6.x/docs/api/child_process.html#child_process_options_stdio @@ -100,15 +91,10 @@ function connectionInfo(ports): ConnectionInfo { }; } -const DEFAULT_PORT_OPTS = { port: 9000, host: "127.0.0.1" } as const; - // gather the connection information for a kernel, write it to a json file, and return it -async function writeConnectionFile(port_options?: { - port?: number; - host?: string; -}) { - const options = { ...DEFAULT_PORT_OPTS, ...port_options }; - const ports = await getPorts(5, options); +async function writeConnectionFile() { + const ports = await getPorts(5); + // console.log("ports = ", ports); // Make sure the kernel runtime dir exists before trying to write the kernel file. const runtimeDir = jupyter_paths.runtimeDir(); @@ -161,11 +147,6 @@ async function launchKernelSpec( ...spawn_options.env, }; - const { execaCommand } = (await dynamicImport( - "execa", - module, - )) as typeof import("execa"); - let running_kernel; if (full_spawn_options.cwd != null) { @@ -180,20 +161,11 @@ async function launchKernelSpec( const escapedCmd = shellEscape(argv); // Prepend the ulimit command - const bashCmd = `${ulimitCmd} && ${escapedCmd}`; + const bashCmd = `${ulimitCmd}\n\n${escapedCmd}`; // Execute the command with ulimit - running_kernel = execaCommand(bashCmd, { - ...full_spawn_options, - shell: true, - }); + running_kernel = await bash(bashCmd, full_spawn_options); } else { - // CRITICAL: I am *NOT* using execa, but instead spawn, because - // I hit bugs in execa. Namely, when argv[0] is a path that doesn't exist, - // no matter what, there is an uncaught exception emitted later. The exact - // same situation with execaCommand or node's spawn does NOT have an uncaught - // exception, so it's a bug. - //running_kernel = execa(argv[0], argv.slice(1), full_spawn_options); // NO! running_kernel = spawn(argv[0], argv.slice(1), full_spawn_options); } diff --git a/src/packages/jupyter/nbgrader/jupyter-run.ts b/src/packages/jupyter/nbgrader/jupyter-run.ts index ebe6093f4f..7b647132b7 100644 --- a/src/packages/jupyter/nbgrader/jupyter-run.ts +++ b/src/packages/jupyter/nbgrader/jupyter-run.ts @@ -3,28 +3,21 @@ * License: MS-RSL – see LICENSE.md for details */ -import type { - JupyterNotebook, - RunNotebookOptions, +import { + type JupyterNotebook, + type RunNotebookOptions, + type Limits, + DEFAULT_LIMITS, } from "@cocalc/util/jupyter/nbgrader-types"; import type { JupyterKernelInterface as JupyterKernel } from "@cocalc/jupyter/types/project-interface"; import { is_object, len, uuid, trunc_middle } from "@cocalc/util/misc"; import { retry_until_success } from "@cocalc/util/async-utils"; import { kernel } from "@cocalc/jupyter/kernel"; import getLogger from "@cocalc/backend/logger"; +export type { Limits }; const logger = getLogger("jupyter:nbgrader:jupyter-run"); -// For tracking limits during the run: -export interface Limits { - timeout_ms_per_cell: number; - max_output_per_cell: number; - max_output: number; - total_output: number; - timeout_ms?: number; - start_time?: number; -} - function global_timeout_exceeded(limits: Limits): boolean { if (limits.timeout_ms == null || limits.start_time == null) return false; return Date.now() - limits.start_time >= limits.timeout_ms; @@ -132,12 +125,13 @@ export async function jupyter_run_notebook( export async function run_cell( jupyter: JupyterKernel, - limits: Limits, + limits0: Partial, cell, ): Promise { if (jupyter == null) { throw Error("jupyter must be defined"); } + const limits = { ...DEFAULT_LIMITS, ...limits0 }; if (limits.timeout_ms && global_timeout_exceeded(limits)) { // the total time has been exceeded -- this will mark outputs as error diff --git a/src/packages/jupyter/package.json b/src/packages/jupyter/package.json index 582336ef2e..e013cacb96 100644 --- a/src/packages/jupyter/package.json +++ b/src/packages/jupyter/package.json @@ -20,27 +20,32 @@ "scripts": { "preinstall": "npx only-allow pnpm", "build": "../node_modules/.bin/tsc --build", - "test": "pnpm exec jest", + "clean": "rm -rf node_modules dist", + "test": "pnpm exec jest --forceExit --maxWorkers=2", + "depcheck": "pnpx depcheck", "tsc": "../node_modules/.bin/tsc --watch --pretty --preserveWatchOutput" }, - "files": ["dist/**", "bin/**", "README.md", "package.json"], + "files": [ + "dist/**", + "bin/**", + "README.md", + "package.json" + ], "author": "SageMath, Inc.", - "keywords": ["cocalc", "jupyter"], + "keywords": [ + "cocalc", + "jupyter" + ], "license": "SEE LICENSE.md", "dependencies": { "@cocalc/backend": "workspace:*", + "@cocalc/conat": "workspace:*", "@cocalc/jupyter": "workspace:*", - "@cocalc/nats": "workspace:*", "@cocalc/sync": "workspace:*", - "@cocalc/sync-client": "workspace:*", "@cocalc/util": "workspace:*", "@nteract/messaging": "^7.0.20", - "@types/json-stable-stringify": "^1.0.32", - "@types/node-cleanup": "^2.1.2", "awaiting": "^3.0.0", "debug": "^4.4.0", - "enchannel-zmq-backend": "^9.1.23", - "execa": "^8.0.1", "expect": "^26.6.2", "he": "^1.2.0", "immutable": "^4.3.0", @@ -52,13 +57,15 @@ "lru-cache": "^7.18.3", "mkdirp": "^1.0.4", "node-cleanup": "^2.1.2", - "portfinder": "^1.0.32", + "rxjs": "^6.6.7", "shell-escape": "^0.2.0", - "tsimportlib": "^0.0.5", - "uuid": "^8.3.2" + "uuid": "^8.3.2", + "zeromq": "^6.4.2" }, "devDependencies": { - "@types/node": "^18.16.14" + "@types/json-stable-stringify": "^1.0.32", + "@types/node": "^18.16.14", + "@types/node-cleanup": "^2.1.2" }, "homepage": "https://github.com/sagemathinc/cocalc/tree/master/src/packages/jupyter", "repository": { diff --git a/src/packages/jupyter/pool/pool.ts b/src/packages/jupyter/pool/pool.ts index f21e810ac2..8e74263a2c 100644 --- a/src/packages/jupyter/pool/pool.ts +++ b/src/packages/jupyter/pool/pool.ts @@ -216,10 +216,7 @@ async function fillWhenEmpty() { // this can definitely throw, e.g., change image and then available kernels change. No need to crash the entire project in that case! await replenishPool(key); } - } catch (error) { - console.log("fillWhenEmpty -- A non-fatal error occurred:", error); - log.error("fillWhenEmpty -- A non-fatal error occurred:", error); - } + } catch {} } async function maintainPool() { diff --git a/src/packages/jupyter/redux/actions.ts b/src/packages/jupyter/redux/actions.ts index eb6bf2d355..e48d938572 100644 --- a/src/packages/jupyter/redux/actions.ts +++ b/src/packages/jupyter/redux/actions.ts @@ -24,7 +24,7 @@ export const MAX_OUTPUT_MESSAGES = 500; // start vanishing from output. Also, this impacts time travel. // WARNING: It is *not* at all difficult to hit fairly large sizes, e.g., 50MB+ // when working with a notebook, by just drawing a bunch of large plots. -const MAX_BLOB_STORE_SIZE = 100 * 1000000; +const MAX_BLOB_STORE_SIZE = 100 * 1e6; declare const localStorage: any; @@ -47,8 +47,8 @@ import { import { SyncDB } from "@cocalc/sync/editor/db/sync"; import type { Client } from "@cocalc/sync/client/types"; import latexEnvs from "@cocalc/util/latex-envs"; -import { jupyterApiClient } from "@cocalc/nats/service/jupyter"; -import { type AKV, akv } from "@cocalc/nats/sync/akv"; +import { jupyterApiClient } from "@cocalc/conat/service/jupyter"; +import { type AKV, akv } from "@cocalc/conat/sync/akv"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; const { close, required, defaults } = misc; @@ -162,8 +162,7 @@ export abstract class JupyterActions extends Actions { return { name: `jupyter:${this.path}`, project_id: this.project_id, - valueType: "binary", - limits: { + config: { max_bytes: MAX_BLOB_STORE_SIZE, }, } as const; @@ -812,7 +811,7 @@ export abstract class JupyterActions extends Actions { // has processed it! const version = this.syncdb.newestVersion(); try { - await this.api({ timeout: 30000 }).save_ipynb_file({ version }); + await this.api({ timeout: 5 * 60 * 1000 }).save_ipynb_file({ version }); } catch (err) { console.log(err); throw Error( diff --git a/src/packages/jupyter/redux/project-actions.ts b/src/packages/jupyter/redux/project-actions.ts index 2ec9faa850..ee1300deec 100644 --- a/src/packages/jupyter/redux/project-actions.ts +++ b/src/packages/jupyter/redux/project-actions.ts @@ -30,9 +30,9 @@ import nbconvertChange from "./handle-nbconvert-change"; import type { ClientFs } from "@cocalc/sync/client/types"; import { kernel as createJupyterKernel } from "@cocalc/jupyter/kernel"; import { removeJupyterRedux } from "@cocalc/jupyter/kernel"; -import { initNatsService } from "@cocalc/jupyter/kernel/nats-service"; -import { type DKV, dkv } from "@cocalc/nats/sync/dkv"; -import { computeServerManager } from "@cocalc/nats/compute/manager"; +import { initConatService } from "@cocalc/jupyter/kernel/conat-service"; +import { type DKV, dkv } from "@cocalc/conat/sync/dkv"; +import { computeServerManager } from "@cocalc/conat/compute/manager"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; // see https://github.com/sagemathinc/cocalc/issues/8060 @@ -175,8 +175,8 @@ export class JupyterActions extends JupyterActions0 { dbg(); this._initialize_manager_already_done = true; - dbg("initialize Jupyter NATS api handler"); - await this.initNatsApi(); + dbg("initialize Jupyter Conat api handler"); + await this.initConatApi(); dbg("initializing blob store"); await this.initBlobStore(); @@ -217,13 +217,13 @@ export class JupyterActions extends JupyterActions0 { ); } - private natsService?; - private initNatsApi = reuseInFlight(async () => { - if (this.natsService != null) { - this.natsService.close(); - this.natsService = null; + private conatService?; + private initConatApi = reuseInFlight(async () => { + if (this.conatService != null) { + this.conatService.close(); + this.conatService = null; } - const service = (this.natsService = await initNatsService({ + const service = (this.conatService = await initConatService({ project_id: this.project_id, path: this.path, })); diff --git a/src/packages/jupyter/stateless-api/execute-stress.test.ts b/src/packages/jupyter/stateless-api/execute-stress.test.ts new file mode 100644 index 0000000000..ce6e8a8747 --- /dev/null +++ b/src/packages/jupyter/stateless-api/execute-stress.test.ts @@ -0,0 +1,18 @@ +import { getPythonKernelName } from "../kernel/kernel-data"; +import jupyterExecute from "./execute"; +import { delay } from "awaiting"; + +const count = 5; +jest.setTimeout(10000); +describe(`execute code ${count} times in a row to test for race conditions`, () => { + // this would randomly hang at one point due to running the init code + // without using the usual execution queue. + it("does the test", async () => { + const kernel = await getPythonKernelName(); + for (let i = 0; i < count; i++) { + const outputs = await jupyterExecute({ kernel, input: "2+3" }); + expect(outputs).toEqual([{ data: { "text/plain": "5" } }]); + await delay(100); + } + }); +}); diff --git a/src/packages/jupyter/stateless-api/execute.test.ts b/src/packages/jupyter/stateless-api/execute.test.ts new file mode 100644 index 0000000000..346ee160dd --- /dev/null +++ b/src/packages/jupyter/stateless-api/execute.test.ts @@ -0,0 +1,48 @@ +import { getPythonKernelName } from "../kernel/kernel-data"; +import jupyterExecute from "./execute"; + +describe("test the jupyterExecute function", () => { + let kernel; + + it(`gets a kernel name`, async () => { + kernel = await getPythonKernelName(); + }); + + it("computes 2+3", async () => { + const outputs = await jupyterExecute({ kernel, input: "a=5; 2+3" }); + expect(outputs).toEqual([{ data: { "text/plain": "5" } }]); + }); + + it("checks that its stateless, i.e., a is not defined", async () => { + const outputs = await jupyterExecute({ kernel, input: "print(a)" }); + expect(JSON.stringify(outputs)).toContain("is not defined"); + }); + + it("sets a via history", async () => { + const outputs = await jupyterExecute({ + kernel, + input: "print(a**2)", + history: ["a=5"], + }); + expect(outputs).toEqual([{ name: "stdout", text: "25\n" }]); + }); + + it("limits the output size", async () => { + const outputs = await jupyterExecute({ + kernel, + input: "print('hi'); import sys; sys.stdout.flush(); print('x'*100)", + limits: { max_output_per_cell: 50 }, + }); + expect(outputs).toEqual([ + { name: "stdout", text: "hi\n" }, + { + name: "stdout", + output_type: "stream", + text: [ + "Output truncated since it exceeded the cell output limit of 50 characters", + ], + }, + ]); + }); +}); + diff --git a/src/packages/jupyter/stateless-api/execute.ts b/src/packages/jupyter/stateless-api/execute.ts index 0c71cb215d..f0865cacfe 100644 --- a/src/packages/jupyter/stateless-api/execute.ts +++ b/src/packages/jupyter/stateless-api/execute.ts @@ -2,26 +2,29 @@ ~/cocalc/src/packages/project$ node Welcome to Node.js v16.19.1. Type ".help" for more information. -> e = require('./dist/jupyter/stateless-api/kernel').default; z = await e.getFromPool('python3'); await z.execute("2+3") + +> e = require('@cocalc/jupyter/stateless-api/execute').default +> await e({input:'2+3',kernel:'python3-ubuntu'}) [ { data: { 'text/plain': '5' } } ] > */ -import { jupyter_execute_response } from "@cocalc/util/message"; import Kernel from "./kernel"; import getLogger from "@cocalc/backend/logger"; +import { type ProjectJupyterApiOptions } from "@cocalc/util/jupyter/api-types"; + const log = getLogger("jupyter:stateless-api:execute"); -export default async function jupyterExecute(socket, mesg) { - log.debug(mesg); +export default async function jupyterExecute(opts: ProjectJupyterApiOptions) { + log.debug(opts); let kernel: undefined | Kernel = undefined; try { - kernel = await Kernel.getFromPool(mesg.kernel, mesg.pool); + kernel = await Kernel.getFromPool(opts.kernel, opts.pool); const outputs: object[] = []; - if (mesg.path != null) { + if (opts.path != null) { try { - await kernel.chdir(mesg.path); + await kernel.chdir(opts.path); log.debug("successful chdir"); } catch (err) { outputs.push({ name: "stderr", text: `${err}` }); @@ -29,20 +32,17 @@ export default async function jupyterExecute(socket, mesg) { } } - if (mesg.history != null && mesg.history.length > 0) { + if (opts.history != null && opts.history.length > 0) { // just execute this directly, since we will ignore the output log.debug("evaluating history"); - await kernel.execute(mesg.history.join("\n"), mesg.limits); + await kernel.execute(opts.history.join("\n"), opts.limits); } - // append the output of running mesg.input to outputs: - for (const output of await kernel.execute(mesg.input, mesg.limits)) { + // append the output of running opts.input to outputs: + for (const output of await kernel.execute(opts.input, opts.limits)) { outputs.push(output); } - socket.write_mesg( - "json", - jupyter_execute_response({ id: mesg.id, output: outputs }) - ); + return outputs; } finally { if (kernel) { await kernel.close(); diff --git a/src/packages/jupyter/stateless-api/kernel-ulimit.test.ts b/src/packages/jupyter/stateless-api/kernel-ulimit.test.ts new file mode 100644 index 0000000000..0efa477c32 --- /dev/null +++ b/src/packages/jupyter/stateless-api/kernel-ulimit.test.ts @@ -0,0 +1,55 @@ +/* +Testing that ulimit is set on the kernels. +*/ + +import { getPythonKernelName } from "../kernel/kernel-data"; +import Kernel from "./kernel"; +import { until } from "@cocalc/util/async-utils"; + +const SECONDS = 2; + +jest.setTimeout(10000); +describe("ulimit is set on the stateless api kernels (and can be configured)", () => { + let kernel; + let kernelName; + + it(`modifies the pool params so the ulimit is ${SECONDS} second of CPU usage`, async () => { + kernelName = await getPythonKernelName(); + Kernel.setUlimit(kernelName, `-t ${SECONDS}`); + }); + + it("gets a kernel", async () => { + // repeat because in rare cases the kernel already in the pool may + // get the ulimit from starting up python (1s of cpu time is short!) + await until( + async () => { + try { + kernel = await Kernel.getFromPool(kernelName); + return true; + } catch { + return false; + } + }, + { start: 1000 }, + ); + }); + + it("quick eval works", async () => { + const output = await kernel.execute("389+11"); + expect(output[0].data["text/plain"]).toBe("400"); + }); + + it(`something that takes infinite CPU time gets killed in at most ${SECONDS} seconds`, async () => { + const start = Date.now(); + try { + await kernel.execute("while True: sum(range(10**8))"); + } catch (err) { + expect(`${err}`).toContain("Kernel last exited with code 137."); + } + expect(Date.now() - start).toBeLessThan(SECONDS * 1500); + }); + + it("cleans up", () => { + kernel.close(); + }); +}); diff --git a/src/packages/jupyter/stateless-api/kernel.ts b/src/packages/jupyter/stateless-api/kernel.ts index f8f8c13099..7eac1324cf 100644 --- a/src/packages/jupyter/stateless-api/kernel.ts +++ b/src/packages/jupyter/stateless-api/kernel.ts @@ -1,30 +1,36 @@ import { kernel as createKernel } from "@cocalc/jupyter/kernel"; import type { JupyterKernelInterface } from "@cocalc/jupyter/types/project-interface"; -import { run_cell, Limits } from "@cocalc/jupyter/nbgrader/jupyter-run"; +import { run_cell } from "@cocalc/jupyter/nbgrader/jupyter-run"; import { mkdtemp, rm } from "fs/promises"; import { tmpdir } from "os"; import { join } from "path"; import getLogger from "@cocalc/backend/logger"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; +import { type Limits } from "@cocalc/util/jupyter/nbgrader-types"; const log = getLogger("jupyter:stateless-api:kernel"); -const DEFAULT_POOL_SIZE = 2; +export const DEFAULT_POOL_SIZE = 2; const DEFAULT_POOL_TIMEOUT_S = 3600; // When we idle timeout we always keep at least this many kernels around. We don't go to 0. const MIN_POOL_SIZE = 1; +// -n = max open files +// -f = max bytes allowed to *write* to disk +// -t = max cputime is 30 seconds +// -v = max virtual memory usage to 3GB +const DEFAULT_ULIMIT = "-n 1000 -f 10485760 -t 30 -v 3000000"; + export default class Kernel { private static pools: { [kernelName: string]: Kernel[] } = {}; private static last_active: { [kernelName: string]: number } = {}; + private static ulimit: { [kernelName: string]: string } = {}; private kernel?: JupyterKernelInterface; private tempDir: string; - constructor(private kernelName: string) { - this.init = reuseInFlight(this.init.bind(this)); - } + constructor(private kernelName: string) {} private static getPool(kernelName: string) { let pool = Kernel.pools[kernelName]; @@ -34,6 +40,11 @@ export default class Kernel { return pool; } + // changing ulimit only impacts NEWLY **created** kernels. + static setUlimit(kernelName: string, ulimit: string) { + Kernel.ulimit[kernelName] = ulimit; + } + // Set a timeout for a given kernel pool (for a specifically named kernel) // to determine when to clear it if no requests have been made. private static setIdleTimeout(kernelName: string, timeout_s: number) { @@ -43,28 +54,31 @@ export default class Kernel { } const now = Date.now(); Kernel.last_active[kernelName] = now; - setTimeout(() => { - if (Kernel.last_active[kernelName] > now) { - // kernel was requested after now. - return; - } - // No recent request for kernelName. - // Keep at least MIN_POOL_SIZE in Kernel.pools[kernelName]. I.e., - // instead of closing and deleting everything, we just want to - // shrink the pool to MIN_POOL_SIZE. - // no request for kernelName, so we clear them from the pool - const poolToShrink = Kernel.pools[kernelName] ?? []; - if (poolToShrink.length > MIN_POOL_SIZE) { - // check if pool needs shrinking - // calculate how many to close - const numToClose = poolToShrink.length - MIN_POOL_SIZE; - for (let i = 0; i < numToClose; i++) { - poolToShrink[i].close(); // close oldest kernels first + setTimeout( + () => { + if (Kernel.last_active[kernelName] > now) { + // kernel was requested after now. + return; } - // update pool to have only the most recent kernels - Kernel.pools[kernelName] = poolToShrink.slice(numToClose); - } - }, (timeout_s ?? DEFAULT_POOL_TIMEOUT_S) * 1000); + // No recent request for kernelName. + // Keep at least MIN_POOL_SIZE in Kernel.pools[kernelName]. I.e., + // instead of closing and deleting everything, we just want to + // shrink the pool to MIN_POOL_SIZE. + // no request for kernelName, so we clear them from the pool + const poolToShrink = Kernel.pools[kernelName] ?? []; + if (poolToShrink.length > MIN_POOL_SIZE) { + // check if pool needs shrinking + // calculate how many to close + const numToClose = poolToShrink.length - MIN_POOL_SIZE; + for (let i = 0; i < numToClose; i++) { + poolToShrink[i].close(); // close oldest kernels first + } + // update pool to have only the most recent kernels + Kernel.pools[kernelName] = poolToShrink.slice(numToClose); + } + }, + (timeout_s ?? DEFAULT_POOL_TIMEOUT_S) * 1000, + ); } static async getFromPool( @@ -72,8 +86,14 @@ export default class Kernel { { size = DEFAULT_POOL_SIZE, timeout_s = DEFAULT_POOL_TIMEOUT_S, - }: { size?: number; timeout_s?: number } = {} + }: { size?: number; timeout_s?: number } = {}, ): Promise { + if (size <= 0) { + // not using a pool -- just create and return kernel + const k = new Kernel(kernelName); + await k.init(); + return k; + } this.setIdleTimeout(kernelName, timeout_s); const pool = Kernel.getPool(kernelName); let i = 1; @@ -83,13 +103,17 @@ export default class Kernel { // we cause this kernel to get init'd soon, but NOT immediately, since starting // several at once just makes them all take much longer exactly when the user // most wants to use their new kernel - setTimeout(async () => { - try { - await k.init(); - } catch (err) { - log.debug("Failed to pre-init Jupyter kernel -- ", kernelName, err); - } - }, 3000 * i); // stagger startup by a few seconds, though kernels that are needed will start ASAP. + setTimeout( + async () => { + try { + await k.init(); + } catch (err) { + log.debug("Failed to pre-init Jupyter kernel -- ", kernelName, err); + } + }, + // stagger startup by a few seconds, though kernels that are needed will start ASAP. + Math.random() * 3000 * i, + ); i += 1; pool.push(k); } @@ -99,39 +123,44 @@ export default class Kernel { return k; } - private async init() { + private init = reuseInFlight(async () => { if (this.kernel != null) { // already initialized return; } this.tempDir = await mkdtemp(join(tmpdir(), "cocalc")); const path = `${this.tempDir}/execute.ipynb`; - // TODO: make this configurable as part of the API call - // I'm having a lot of trouble with this for now. - // -n = max open files - // -f = max bytes allowed to *write* to disk - // -t = max cputime is 30 seconds - // -v = max virtual memory usage to 3GB this.kernel = createKernel({ name: this.kernelName, path, - // ulimit: `-n 1000 -f 10485760 -t 30 -v 3000000`, + ulimit: Kernel.ulimit[this.kernelName] ?? DEFAULT_ULIMIT, }); await this.kernel.ensure_running(); await this.kernel.execute_code_now({ code: "" }); + }); + + // empty all pools and do not refill + static closeAll() { + for (const kernelName in Kernel.pools) { + for (const kernel of Kernel.pools[kernelName]) { + kernel.close(); + } + } + Kernel.pools = {}; + Kernel.last_active = {}; } - async execute( + execute = async ( code: string, - limits: Limits = { + limits: Partial = { timeout_ms: 30000, timeout_ms_per_cell: 30000, max_output: 5000000, max_output_per_cell: 1000000, start_time: Date.now(), total_output: 0, - } - ) { + }, + ) => { if (this.kernel == null) { throw Error("kernel already closed"); } @@ -142,23 +171,26 @@ export default class Kernel { const cell = { cell_type: "code", source: [code], outputs: [] }; await run_cell(this.kernel, limits, cell); return cell.outputs; - } + }; - async chdir(path: string) { + chdir = async (path: string) => { if (this.kernel == null) return; await this.kernel.chdir(path); - } + }; - async returnToPool(): Promise { + // this is not used anywhere + returnToPool = async (): Promise => { if (this.kernel == null) { throw Error("kernel already closed"); } const pool = Kernel.getPool(this.kernelName); pool.push(this); - } + }; - async close() { - if (this.kernel == null) return; + close = async () => { + if (this.kernel == null) { + return; + } try { await this.kernel.close(); } catch (err) { @@ -171,5 +203,5 @@ export default class Kernel { } catch (err) { log.warn("Error cleaning up temporary directory", err); } - } + }; } diff --git a/src/packages/jupyter/stateless-api/stateless-api-kernel.test.ts b/src/packages/jupyter/stateless-api/stateless-api-kernel.test.ts new file mode 100644 index 0000000000..55dd9597d7 --- /dev/null +++ b/src/packages/jupyter/stateless-api/stateless-api-kernel.test.ts @@ -0,0 +1,88 @@ +/* +Test the Jupyer stateless api kernel functionality. +*/ + +import { getPythonKernelName } from "../kernel/kernel-data"; +import Kernel, { DEFAULT_POOL_SIZE } from "./kernel"; + +describe("create a jupyter stateless-api kernel and test basic functionality", () => { + let kernel; + it("gets a kernel", async () => { + const kernelName = await getPythonKernelName(); + // @ts-ignore + expect(Kernel.getPool(kernelName).length).toBe(0); + kernel = await Kernel.getFromPool(kernelName); + // @ts-ignore + expect(Kernel.getPool(kernelName).length).toBe(DEFAULT_POOL_SIZE); + }); + + it("confirms it is 'running'", () => { + expect(kernel.kernel.get_state()).toBe("running"); + }); + + it("compute 2+3", async () => { + const output = await kernel.execute("2+3"); + expect(output[0].data["text/plain"]).toBe("5"); + }); + + it("exec something with two distinct output messages", async () => { + const output = await kernel.execute( + "import sys; sys.stdout.write('1'); sys.stdout.flush(); print('2')", + ); + expect(output.length).toBe(2); + expect(output).toEqual([ + { name: "stdout", text: "1" }, + { name: "stdout", text: "2\n" }, + ]); + }); + + it("exec something that throws an error", async () => { + const output = await kernel.execute("1/0"); + expect(output[0].traceback.join("")).toContain("division by zero"); + }); + + it("chdir to /tmp and confirm that", async () => { + await kernel.chdir("/tmp"); + const output = await kernel.execute( + "import os; os.path.abspath(os.curdir)", + ); + expect(output).toEqual([{ data: { "text/plain": "'/tmp'" } }]); + }); + + it("get another kernel and confirm pool is maintained", async () => { + const kernelName = await getPythonKernelName(); + const kernel2 = await Kernel.getFromPool(kernelName); + // @ts-ignore + expect(Kernel.getPool(kernelName).length).toBe(DEFAULT_POOL_SIZE); + kernel2.close(); + }); + + it("cleans up", () => { + kernel.close(); + Kernel.closeAll(); + }); +}); + +describe("test timeout - this is how long until pool starts getting trimmed", () => { + let kernel; + it("gets a kernel with a 1s timeout", async () => { + const kernelName = await getPythonKernelName(); + kernel = await Kernel.getFromPool(kernelName, { timeout_s: 1 }); + }); + + it("quick eval works", async () => { + const output = await kernel.execute("389+11"); + expect(output[0].data["text/plain"]).toBe("400"); + }); + + it("something that takes more than a second", async () => { + await kernel.execute("print('hi'); import time; time.sleep(1.2)"); + }); + + it("now check that the pool started shrinking", async () => { + const kernelName = await getPythonKernelName(); + // @ts-ignore + expect(Kernel.getPool(kernelName).length).toBeLessThan(DEFAULT_POOL_SIZE); + }); +}); + diff --git a/src/packages/jupyter/tsconfig.json b/src/packages/jupyter/tsconfig.json index 56e63b5680..743141e8d2 100644 --- a/src/packages/jupyter/tsconfig.json +++ b/src/packages/jupyter/tsconfig.json @@ -8,8 +8,7 @@ "references": [ { "path": "../backend" }, { "path": "../sync" }, - { "path": "../sync-client" }, { "path": "../util" }, - { "path": "../nats" } + { "path": "../conat" } ] } diff --git a/src/packages/jupyter/types/project-interface.ts b/src/packages/jupyter/types/project-interface.ts index f87c5690cb..495c285a49 100644 --- a/src/packages/jupyter/types/project-interface.ts +++ b/src/packages/jupyter/types/project-interface.ts @@ -14,6 +14,7 @@ so that Typescript can meaningfully type check everything. import type { Channels } from "@nteract/messaging"; import type { KernelInfo } from "@cocalc/util/jupyter/types"; export type { KernelInfo }; +import type { EventIterator } from "@cocalc/util/event-iterator"; // see https://gist.github.com/rsms/3744301784eb3af8ed80bc746bef5eeb#file-eventlistener-d-ts export interface EventEmitterInterface { @@ -75,12 +76,16 @@ export interface ExecOpts { timeout_ms?: number; } +export type OutputMessage = object; // todo + export interface CodeExecutionEmitterInterface extends EventEmitterInterface { - emit_output(result: object): void; + emit_output(result: OutputMessage): void; cancel(): void; close(): void; throw_error(err): void; - go(): Promise; + go(): Promise; + iter(): EventIterator; + waitUntilDone: () => Promise; } interface JupyterKernelInterfaceSpawnOpts { @@ -95,7 +100,7 @@ export interface JupyterKernelInterface extends EventEmitterInterface { get_state(): string; signal(signal: string): void; - close(): Promise; + close(): void; spawn(opts?: JupyterKernelInterfaceSpawnOpts): Promise; execute_code(opts: ExecOpts): CodeExecutionEmitterInterface; cancel_execute(id: string): void; diff --git a/src/packages/jupyter/zmq/jmp.ts b/src/packages/jupyter/zmq/jmp.ts new file mode 100644 index 0000000000..dd25b0ecf4 --- /dev/null +++ b/src/packages/jupyter/zmq/jmp.ts @@ -0,0 +1,367 @@ +/* +This is from https://github.com/n-riesco/jmp but rewritten in typescript. + +The original and all modifications in CoCalc of the code in THIS DIRECTORY +are: * BSD 3-Clause License * + +*/ + +/* + * BSD 3-Clause License + * + * Copyright (c) 2015, Nicolas Riesco and others as credited in the AUTHORS file + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + */ + +import crypto from "crypto"; +import { v4 as uuid } from "uuid"; +import * as zmq from "zeromq/v5-compat"; + +const DEBUG = (global as any).DEBUG || false; + +let log: (...args: any[]) => void; +if (DEBUG) { + // eslint-disable-next-line @typescript-eslint/no-var-requires + const console = require("console"); + log = (...args) => { + process.stderr.write("JMP: "); + console.error(...args); + }; +} else { + try { + // eslint-disable-next-line @typescript-eslint/no-var-requires + log = require("debug")("JMP:"); + } catch { + log = () => {}; + } +} + +export const DELIMITER = ""; + +export interface JupyterHeader { + msg_id?: string; + username?: string; + session?: string; + msg_type?: string; + version?: string; + [key: string]: any; +} + +export interface MessageProps { + idents?: Buffer[]; + header?: JupyterHeader; + parent_header?: JupyterHeader; + metadata?: any; + content?: any; + buffers?: Buffer[]; +} + +export class Message { + idents: Buffer[]; + header: JupyterHeader; + parent_header: JupyterHeader; + metadata: object; + content: object; + buffers: Buffer[]; + + constructor(properties?: MessageProps) { + this.idents = properties?.idents ?? []; + this.header = properties?.header ?? {}; + this.parent_header = properties?.parent_header ?? {}; + this.metadata = properties?.metadata ?? {}; + this.content = properties?.content ?? {}; + this.buffers = properties?.buffers ?? []; + } + + respond( + socket: zmq.Socket, + messageType: string, + content?: object, + metadata?: object, + protocolVersion?: string, + ): Message { + const response = new Message(); + response.idents = this.idents.slice(); + response.header = { + msg_id: uuid(), + username: this.header.username, + session: this.header.session, + msg_type: messageType, + }; + if (this.header?.version) { + response.header.version = this.header.version; + } + if (protocolVersion) { + response.header.version = protocolVersion; + } + response.parent_header = { ...this.header }; + response.content = content ?? {}; + response.metadata = metadata ?? {}; + socket.send(response as any); + return response; + } + + static _decode( + messageFrames: Buffer[] | IArguments, + scheme = "sha256", + key = "", + ): Message | null { + try { + return _decode(messageFrames, scheme, key); + } catch (err) { + log("MESSAGE: DECODE: Error:", err); + return null; + } + } + + _encode(scheme = "sha256", key = ""): (Buffer | string)[] { + const idents = this.idents; + + const header = JSON.stringify(this.header); + const parent_header = JSON.stringify(this.parent_header); + const metadata = JSON.stringify(this.metadata); + const content = JSON.stringify(this.content); + + let signature = ""; + if (key) { + const hmac = crypto.createHmac(scheme, key); + const encoding = "utf8"; + hmac.update(Buffer.from(header, encoding)); + hmac.update(Buffer.from(parent_header, encoding)); + hmac.update(Buffer.from(metadata, encoding)); + hmac.update(Buffer.from(content, encoding)); + signature = hmac.digest("hex"); + } + + return [ + ...idents, + DELIMITER, + signature, + header, + parent_header, + metadata, + content, + ...this.buffers, + ]; + } +} + +// Helper decode +function _decode( + messageFrames: Buffer[] | IArguments, + scheme: string, + key: string, +): Message | null { + // Could be an arguments object, convert to array if so + const frames = Array.isArray(messageFrames) + ? messageFrames + : Array.prototype.slice.call(messageFrames); + + let i = 0; + const idents: Buffer[] = []; + for (; i < frames.length; i++) { + const frame = frames[i]; + if (frame.toString() === DELIMITER) break; + idents.push(frame); + } + + if (frames.length - i < 5) { + log("MESSAGE: DECODE: Not enough message frames", frames); + return null; + } + + if (frames[i].toString() !== DELIMITER) { + log("MESSAGE: DECODE: Missing delimiter", frames); + return null; + } + + if (key) { + const obtainedSignature = frames[i + 1].toString(); + const hmac = crypto.createHmac(scheme, key); + hmac.update(frames[i + 2]); + hmac.update(frames[i + 3]); + hmac.update(frames[i + 4]); + hmac.update(frames[i + 5]); + const expectedSignature = hmac.digest("hex"); + + if (expectedSignature !== obtainedSignature) { + log( + "MESSAGE: DECODE: Incorrect message signature:", + "Obtained =", + obtainedSignature, + "Expected =", + expectedSignature, + ); + return null; + } + } + + return new Message({ + idents: idents, + header: JSON.parse(frames[i + 2].toString()), + parent_header: JSON.parse(frames[i + 3].toString()), + metadata: JSON.parse(frames[i + 4].toString()), + content: JSON.parse(frames[i + 5].toString()), + buffers: frames.slice(i + 6), + }); +} + +// Socket + +export class Socket extends zmq.Socket { + _jmp: { + scheme: string; + key: string; + _listeners: { + unwrapped: (...args: any[]) => void; + wrapped: (...args: any[]) => void; + }[]; + }; + + constructor(socketType: zmq.SocketType, scheme = "sha256", key = "") { + super(socketType); + this._jmp = { + scheme, + key, + _listeners: [], + }; + } + + // @ts-ignore + send( + message: Message | string | Buffer | (Message | Buffer | string)[], + flags?: number, + ): this { + const p = Object.getPrototypeOf(Socket.prototype); + + if (message instanceof Message) { + log("SOCKET: SEND:", message); + // @ts-ignore + p.send.call( + this, + message._encode(this._jmp.scheme, this._jmp.key), + flags, + ); + return this; + } + // @ts-ignore + p.send.apply(this, arguments); + return this; + } + + on(event: string, listener: (...args: any[]) => void): this { + const p = Object.getPrototypeOf(Socket.prototype); + if (event !== "message") { + // @ts-ignore + p.on.apply(this, arguments); + return this; + } + + const _listener = { + unwrapped: listener, + wrapped: ((...args: any[]) => { + const message = Message._decode(args, this._jmp.scheme, this._jmp.key); + if (message) { + listener(message); + } + }).bind(this), + }; + this._jmp._listeners.push(_listener); + // @ts-ignore + p.on.call(this, event, _listener.wrapped); + return this; + } + + addListener = this.on; + + once(event: string, listener: (...args: any[]) => void): this { + const p = Object.getPrototypeOf(Socket.prototype); + if (event !== "message") { + // @ts-ignore + p.once.apply(this, arguments); + return this; + } + + const _listener = { + unwrapped: listener, + wrapped: ((...args: any[]) => { + const message = Message._decode(args, this._jmp.scheme, this._jmp.key); + if (message) { + try { + listener(message); + } catch (error) { + this.removeListener(event, listener); + throw error; + } + } + this.removeListener(event, listener); + }).bind(this), + }; + this._jmp._listeners.push(_listener); + // @ts-ignore + p.on.call(this, event, _listener.wrapped); + return this; + } + + removeListener(event: string, listener: (...args: any[]) => void): this { + const p = Object.getPrototypeOf(Socket.prototype); + if (event !== "message") { + // @ts-ignore + p.removeListener.apply(this, arguments); + return this; + } + + const index = this._jmp._listeners.findIndex( + (l) => l.unwrapped === listener, + ); + if (index !== -1) { + const _listener = this._jmp._listeners[index]; + this._jmp._listeners.splice(index, 1); + // @ts-ignore + p.removeListener.call(this, event, _listener.wrapped); + return this; + } + // @ts-ignore + p.removeListener.apply(this, arguments); + return this; + } + + removeAllListeners(event?: string): this { + const p = Object.getPrototypeOf(Socket.prototype); + if (!event || event === "message") { + this._jmp._listeners.length = 0; + } + // @ts-ignore + p.removeAllListeners.apply(this, arguments); + return this; + } +} + +export { zmq }; diff --git a/src/packages/jupyter/zmq/jupyter-channels.ts b/src/packages/jupyter/zmq/jupyter-channels.ts new file mode 100644 index 0000000000..0fa9e49b08 --- /dev/null +++ b/src/packages/jupyter/zmq/jupyter-channels.ts @@ -0,0 +1,266 @@ +/* +This is adapted from https://github.com/nteract/enchannel-zmq-backend but with +significant rewriting, bug fixing, etc. + +The original and all modifications in CoCalc of the code in THIS DIRECTORY +are: * BSD 3-Clause License * +*/ + +import { Channels, JupyterMessage } from "@nteract/messaging"; +import * as moduleJMP from "./jmp"; +import { fromEvent, merge, Observable, Subject, Subscriber } from "rxjs"; +import { FromEventTarget } from "rxjs/internal/observable/fromEvent"; +import { map, publish, refCount } from "rxjs/operators"; +import { v4 as uuid } from "uuid"; +import { once } from "@cocalc/util/async-utils"; + +export const ZMQType = { + frontend: { + iopub: "sub", + stdin: "dealer", + shell: "dealer", + control: "dealer", + }, +} as const; + +type ChannelName = "iopub" | "stdin" | "shell" | "control"; + +export interface JupyterConnectionInfo { + version: number; + iopub_port: number; + shell_port: number; + stdin_port: number; + control_port: number; + signature_scheme: "hmac-sha256"; + hb_port: number; + ip: string; + key: string; + transport: "tcp" | "ipc"; +} + +interface HeaderFiller { + session: string; + username: string; +} + +/** + * Takes a Jupyter spec connection info object and channel and returns the + * string for a channel. Abstracts away tcp and ipc connection string + * formatting + * + * @param config Jupyter connection information + * @param channel Jupyter channel ("iopub", "shell", "control", "stdin") + * + * @returns The connection string + */ +export const formConnectionString = ( + config: JupyterConnectionInfo, + channel: ChannelName, +) => { + const portDelimiter = config.transport === "tcp" ? ":" : "-"; + const port = config[`${channel}_port` as keyof JupyterConnectionInfo]; + if (!port) { + throw new Error(`Port not found for channel "${channel}"`); + } + return `${config.transport}://${config.ip}${portDelimiter}${port}`; +}; + +/** + * Creates a socket for the given channel with ZMQ channel type given a config + * + * @param channel Jupyter channel ("iopub", "shell", "control", "stdin") + * @param config Jupyter connection information + * + * @returns The new Jupyter ZMQ socket + */ +async function createSocket( + channel: ChannelName, + config: JupyterConnectionInfo, + jmp = moduleJMP, +): Promise { + const zmqType = ZMQType.frontend[channel]; + const scheme = config.signature_scheme.slice("hmac-".length); + const socket = new jmp.Socket(zmqType, scheme, config.key); + const url = formConnectionString(config, channel); + const connected = once(socket, "connect"); + socket.monitor(); + socket.connect(url); + await connected; + return socket; +} + +export const getUsername = () => + process.env.LOGNAME || + process.env.USER || + process.env.LNAME || + process.env.USERNAME || + "username"; // This is the fallback that the classic notebook uses + +/** + * Creates a multiplexed set of channels. + * + * @param config Jupyter connection information + * @param config.ip IP address of the kernel + * @param config.transport Transport, e.g. TCP + * @param config.signature_scheme Hashing scheme, e.g. hmac-sha256 + * @param config.iopub_port Port for iopub channel + * @param subscription subscribed topic; defaults to all + * + * @returns Subject containing multiplexed channels + */ +export const createMainChannel = async ( + config: JupyterConnectionInfo, + identity: string, + subscription: string = "", + header: HeaderFiller = { + session: uuid(), + username: getUsername(), + }, + jmp = moduleJMP, +): Promise => { + const sockets = await createSockets(config, subscription, jmp); + allSockets[identity] = sockets; + const main = createMainChannelFromSockets(sockets, header, jmp); + return main; +}; + +const allSockets: { [identity: string]: any } = {}; + +export function closeSockets(identity: string) { + const x = allSockets[identity]; + if (x != null) { + for (const name in x) { + x[name].close(); + } + } + delete allSockets[identity]; +} + +/** + * Sets up the sockets for each of the jupyter channels. + * + * @param config Jupyter connection information + * @param subscription The topic to filter the subscription to the iopub channel on + * @param jmp A reference to the JMP Node module + * + * @returns Sockets for each Jupyter channel + */ +export const createSockets = async ( + config: JupyterConnectionInfo, + subscription: string = "", + jmp = moduleJMP, +) => { + const [shell, control, stdin, iopub] = await Promise.all([ + createSocket("shell", config, jmp), + createSocket("control", config, jmp), + createSocket("stdin", config, jmp), + createSocket("iopub", config, jmp), + ]); + + // NOTE: ZMQ PUB/SUB subscription (not an Rx subscription) + iopub.subscribe(subscription); + + return { + shell, + control, + stdin, + iopub, + }; +}; + +/** + * Creates a multiplexed set of channels. + * + * @param sockets An object containing associations between channel types and 0MQ sockets + * @param header The session and username to place in kernel message headers + * @param jmp A reference to the JMP Node module + * + * @returns Creates an Observable for each channel connection that allows us + * to send and receive messages through the Jupyter protocol. + */ +export const createMainChannelFromSockets = ( + sockets: { + [name: string]: moduleJMP.Socket; + }, + header: HeaderFiller = { + session: uuid(), + username: getUsername(), + }, + jmp = moduleJMP, +): Channels => { + // The mega subject that encapsulates all the sockets as one multiplexed + // stream + + const outgoingMessages = Subscriber.create( + (message) => { + // There's always a chance that a bad message is sent, we'll ignore it + // instead of consuming it + if (!message || !message.channel) { + console.warn("message sent without a channel", message); + return; + } + const socket = sockets[message.channel]; + if (!socket) { + // If, for some reason, a message is sent on a channel we don't have + // a socket for, warn about it but don't bomb the stream + console.warn("channel not understood for message", message); + return; + } + try { + const jMessage = new jmp.Message({ + // Fold in the setup header to ease usage of messages on channels + header: { ...message.header, ...header }, + parent_header: message.parent_header, + content: message.content, + metadata: message.metadata, + buffers: message.buffers as any, + }); + socket.send(jMessage); + } catch (err) { + console.error("Error sending message", err, message); + } + }, + undefined, // not bothering with sending errors on + () => + // When the subject is completed / disposed, close all the event + // listeners and shutdown the socket + Object.keys(sockets).forEach((name) => { + const socket = sockets[name]; + socket.removeAllListeners(); + socket.close?.(); + }), + ); + + // Messages from kernel on the sockets + const incomingMessages: Observable = merge( + // Form an Observable with each socket + ...Object.keys(sockets).map((name) => { + const socket = sockets[name]; + // fromEvent typings are broken. socket will work as an event target. + return fromEvent( + // Pending a refactor around jmp, this allows us to treat the socket + // as a normal event emitter + socket as unknown as FromEventTarget, + "message", + ).pipe( + map((body: JupyterMessage): JupyterMessage => { + // Route the message for the frontend by setting the channel + const msg = { ...body, channel: name }; + // Conform to same message format as notebook websockets + // See https://github.com/n-riesco/jmp/issues/10 + delete (msg as any).idents; + return msg; + }), + publish(), + refCount(), + ); + }), + ).pipe(publish(), refCount()); + + const subject: Subject = Subject.create( + outgoingMessages, + incomingMessages, + ); + + return subject; +}; diff --git a/src/packages/nats/README.md b/src/packages/nats/README.md deleted file mode 100644 index 62427f1bab..0000000000 --- a/src/packages/nats/README.md +++ /dev/null @@ -1,9 +0,0 @@ -### NOTES - -- We are using base64 encoding of keys because we want to allow arbitrary kesy but nats can't. NOTE: at one point this code used sha1 hashes for keys, but we switche to base64 because: (1) base64 is smaller for shorter keys and many keys are short, (2) computing base64 of a string is MUCH faster than computing sha1, (3) this completely eliminates any worry about hash collisions, and (4) it was easy to patch Nats to allow any base64 string (see below). - -- We have to store various data, e.g., arbitrary file paths, as nats segments, so MUST have a string with allowed characters. - - - NATS officially allows: "Any Unicode character except null, space, ., \* and >" - See https://docs.nats.io/nats-concepts/subjects#characters-allowed-and-recommended-for-subject-names - - However, the nats kv javascript client uses an incredibly restrictive key checker, which doesn't even allow base64 encoding! Geez: https://github.com/nats-io/nats.js/issues/246 diff --git a/src/packages/nats/browser-api/index.ts b/src/packages/nats/browser-api/index.ts deleted file mode 100644 index 16c9326856..0000000000 --- a/src/packages/nats/browser-api/index.ts +++ /dev/null @@ -1,48 +0,0 @@ -/* -Request/response API that runs in each browser client. - -DEVELOPMENT: - -Refresh your browser and do this in the console to connect to your own browser: - - > a = cc.client.nats_client.browserApi({sessionId:cc.client.nats_client.sessionId}) - -Then try everything. - -You can also open a second browser tab (with the same account), view the sessionId - - > cc.client.nats_client.sessionId - -then connect from one to the other using that sessionId. This way you can coordinate -between different browsers. -*/ - -import { type System, system } from "./system"; -import { handleErrorMessage } from "@cocalc/nats/util"; - -export interface BrowserApi { - system: System; -} - -const BrowserApiStructure = { - system, -} as const; - -export function initBrowserApi(callBrowserApi): BrowserApi { - const browserApi: any = {}; - for (const group in BrowserApiStructure) { - if (browserApi[group] == null) { - browserApi[group] = {}; - } - for (const functionName in BrowserApiStructure[group]) { - browserApi[group][functionName] = async (...args) => - handleErrorMessage( - await callBrowserApi({ - name: `${group}.${functionName}`, - args, - }), - ); - } - } - return browserApi as BrowserApi; -} diff --git a/src/packages/nats/browser-api/system.ts b/src/packages/nats/browser-api/system.ts deleted file mode 100644 index 155a87e344..0000000000 --- a/src/packages/nats/browser-api/system.ts +++ /dev/null @@ -1,9 +0,0 @@ -export const system = { - version: true, - ping: true, -}; - -export interface System { - version: () => Promise; - ping: () => Promise<{ now: number }>; -} diff --git a/src/packages/nats/changefeed/client.ts b/src/packages/nats/changefeed/client.ts deleted file mode 100644 index b5d16955b4..0000000000 --- a/src/packages/nats/changefeed/client.ts +++ /dev/null @@ -1,98 +0,0 @@ -/* -Changefeed client - - -changefeed(...) -- returns async iterator that outputs: - - - {id:string} -- defines id of the changefeed - - '' -- heartbeats - - standard changefeed messages exactly from the database - -renew({id, lifetime}) -- keeps the changefeed alive for at least lifetime more ms. -*/ - -import { getEnv } from "@cocalc/nats/client"; -import { isValidUUID } from "@cocalc/util/misc"; -import { changefeedSubject, renewSubject, LAST_CHUNK } from "./server"; -import { waitUntilConnected } from "@cocalc/nats/util"; -export { DEFAULT_LIFETIME } from "./server"; - -export async function* changefeed({ - account_id, - query, - options, - heartbeat, - lifetime, - maxActualLifetime = 1000 * 60 * 60 * 2, -}: { - account_id: string; - query: any; - options?: any[]; - // maximum amount of time the changefeed can possibly stay alive, even with - // many calls to extend it. - maxActualLifetime?: number; - // server will send resp='' to ensure there is at least one message every this many ms. - heartbeat?: number; - // changefeed will live at most this long, then definitely die. - lifetime?: number; -}) { - if (!isValidUUID(account_id)) { - throw Error("account_id must be a valid uuid"); - } - const subject = changefeedSubject({ account_id }); - - let lastSeq = -1; - const { nc, jc } = await getEnv(); - await waitUntilConnected(); - const chunks: Uint8Array[] = []; - for await (const mesg of await nc.requestMany( - subject, - jc.encode({ query, options, heartbeat, lifetime }), - { maxWait: maxActualLifetime }, - )) { - if (mesg.data.length == 0) { - // done - return; - } - chunks.push(mesg.data); - if (!isLastChunk(mesg)) { - continue; - } - const data = Buffer.concat(chunks); - chunks.length = 0; - const { error, resp, seq } = jc.decode(data); - if (error) { - throw Error(error); - } - if (lastSeq + 1 != seq) { - throw Error("missed response"); - } - lastSeq = seq; - yield resp; - } -} - -function isLastChunk(mesg) { - for (const [key, _] of mesg.headers ?? []) { - if (key == LAST_CHUNK) { - return true; - } - } - return false; -} - -export async function renew({ - account_id, - id, - lifetime, -}: { - account_id: string; - id: string; - lifetime?: number; -}) { - const subject = renewSubject({ account_id }); - const { nc, jc } = await getEnv(); - await waitUntilConnected(); - const resp = await nc.request(subject, jc.encode({ id, lifetime })); - return jc.decode(resp.data); -} diff --git a/src/packages/nats/changefeed/server.ts b/src/packages/nats/changefeed/server.ts deleted file mode 100644 index 1ed0263925..0000000000 --- a/src/packages/nats/changefeed/server.ts +++ /dev/null @@ -1,327 +0,0 @@ -/* -Multiresponse request/response NATS changefeed server. - -- Chunking to arbitrarily large data -- Lifetimes that can be extended -- Heartbeats -*/ - -import { getEnv } from "@cocalc/nats/client"; -import { type Subscription, Empty, headers } from "@nats-io/nats-core"; -import { isValidUUID, uuid } from "@cocalc/util/misc"; -import { getLogger } from "@cocalc/nats/client"; -import { waitUntilConnected } from "@cocalc/nats/util"; -import { delay } from "awaiting"; -import { getMaxPayload } from "@cocalc/nats/util"; - -export const DEFAULT_LIFETIME = 1000 * 60; -export const MAX_LIFETIME = 15 * 1000 * 60; -export const MIN_LIFETIME = 30 * 1000; -export const MIN_HEARTBEAT = 5000; -export const MAX_HEARTBEAT = 120000; -export const MAX_CHANGEFEEDS_PER_ACCOUNT = parseInt( - process.env.MAX_CHANGEFEEDS_PER_ACCOUNT ?? "100", -); - -export const MAX_CHANGEFEEDS_PER_SERVER = parseInt( - process.env.MAX_CHANGEFEEDS_PER_SERVER ?? "5000", -); - -export const LAST_CHUNK = "last-chunk"; - -const logger = getLogger("changefeed:server"); - -export const SUBJECT = process.env.COCALC_TEST_MODE - ? "changefeeds-test" - : "changefeeds"; - -export function changefeedSubject({ account_id }: { account_id: string }) { - return `${SUBJECT}.account-${account_id}.api`; -} - -export function renewSubject({ account_id }: { account_id: string }) { - return `${SUBJECT}.account-${account_id}.renew`; -} - -function getUserId(subject: string): string { - if (subject.startsWith(`${SUBJECT}.account-`)) { - return subject.slice( - `${SUBJECT}.account-`.length, - `${SUBJECT}.account-`.length + 36, - ); - } - throw Error("invalid subject"); -} - -let terminated = false; -let sub: Subscription | null = null; -export async function init(db) { - logger.debug("starting changefeed server"); - logger.debug({ - DEFAULT_LIFETIME, - MAX_LIFETIME, - MIN_LIFETIME, - MIN_HEARTBEAT, - MAX_HEARTBEAT, - MAX_CHANGEFEEDS_PER_ACCOUNT, - MAX_CHANGEFEEDS_PER_SERVER, - SUBJECT, - }); - changefeedMainLoop(db); - renewMainLoop(); -} - -async function changefeedMainLoop(db) { - while (!terminated) { - await waitUntilConnected(); - const { nc } = await getEnv(); - sub = nc.subscribe(`${SUBJECT}.*.api`, { queue: "q" }); - try { - await listen(db); - } catch (err) { - logger.debug(`WARNING: changefeedMainLoop error -- ${err}`); - } - await delay(15000); - } -} - -let renew: Subscription | null = null; -async function renewMainLoop() { - while (!terminated) { - await waitUntilConnected(); - const { nc } = await getEnv(); - renew = nc.subscribe(`${SUBJECT}.*.renew`); - try { - await listenRenew(); - } catch (err) { - logger.debug(`WARNING: renewMainLoop error -- ${err}`); - } - await delay(15000); - } -} - -async function listenRenew() { - if (renew == null) { - throw Error("must call init first"); - } - for await (const mesg of renew) { - if (terminated) { - return; - } - handleRenew(mesg); - } -} - -const endOfLife: { [id: string]: number } = {}; -function getLifetime({ lifetime }): number { - if (lifetime === -1) { - // special case of -1 used for cancel - return lifetime; - } - if (!lifetime) { - return DEFAULT_LIFETIME; - } - lifetime = parseFloat(lifetime); - if (lifetime > MAX_LIFETIME) { - return MAX_LIFETIME; - } - if (lifetime < MIN_LIFETIME) { - return MIN_LIFETIME; - } - return lifetime; -} - -async function handleRenew(mesg) { - const { jc } = await getEnv(); - const request = jc.decode(mesg.data); - if (!request) { - return; - } - let { id } = request; - if (endOfLife[id]) { - // it's ours so we respond - const lifetime = getLifetime(request); - endOfLife[id] = Date.now() + lifetime; - mesg.respond(jc.encode({ status: "ok" })); - } -} - -export async function terminate() { - terminated = true; - if (sub != null) { - sub.drain(); - sub = null; - } - if (renew != null) { - renew.drain(); - renew = null; - } -} - -async function listen(db) { - if (sub == null) { - throw Error("must call init first"); - } - for await (const mesg of sub) { - if (terminated) { - return; - } - handleMessage(mesg, db); - } -} - -let numChangefeeds = 0; -const numChangefeedsPerAccount: { [account_id: string]: number } = {}; - -function metrics() { - logger.debug("changefeeds", { numChangefeeds }); -} - -async function send({ jc, mesg, resp }) { - const maxPayload = (await getMaxPayload()) - 1000; // slack for header - const data = jc.encode(resp); - const chunks: Buffer[] = []; - for (let i = 0; i < data.length; i += maxPayload) { - const slice = data.slice(i, i + maxPayload); - chunks.push(slice); - } - if (chunks.length > 1) { - logger.debug(`sending message with ${chunks.length} chunks`); - } - for (let i = 0; i < chunks.length; i++) { - if (i == chunks.length - 1) { - const h = headers(); - h.append(LAST_CHUNK, "true"); - mesg.respond(chunks[i], { headers: h }); - } else { - mesg.respond(chunks[i]); - } - } -} - -async function handleMessage(mesg, db) { - const { jc } = await getEnv(); - const request = jc.decode(mesg.data); - const account_id = getUserId(mesg.subject); - const id = uuid(); - - let seq = 0; - let lastSend = 0; - const respond = async (error, resp?) => { - if (terminated) { - end(); - } - lastSend = Date.now(); - if (resp?.action == "close") { - end(); - } else { - await send({ jc, mesg, resp: { resp, error, seq } }); - seq += 1; - if (error) { - end(); - } - } - }; - - numChangefeeds += 1; - metrics(); - let done = false; - const end = () => { - if (done) { - return; - } - done = true; - delete endOfLife[id]; - numChangefeeds -= 1; - metrics(); - db().user_query_cancel_changefeed({ id }); - // end response stream with empty payload. - mesg.respond(Empty); - }; - - if (numChangefeedsPerAccount[account_id] > MAX_CHANGEFEEDS_PER_ACCOUNT) { - logger.debug( - `numChangefeedsPerAccount[${account_id}] >= MAX_CHANGEFEEDS_PER_ACCOUNT`, - { - numChangefeedsPerAccountThis: numChangefeedsPerAccount[account_id], - MAX_CHANGEFEEDS_PER_ACCOUNT, - }, - ); - respond( - `This server has a limit of ${MAX_CHANGEFEEDS_PER_ACCOUNT} changefeeds per account`, - ); - return; - } - if (numChangefeeds >= MAX_CHANGEFEEDS_PER_SERVER) { - logger.debug("numChangefeeds >= MAX_CHANGEFEEDS_PER_SERVER", { - numChangefeeds, - MAX_CHANGEFEEDS_PER_SERVER, - }); - // this will just cause the client to make another attempt, hopefully - // to another server - respond( - `This server has a limit of ${MAX_CHANGEFEEDS_PER_SERVER} changefeeds`, - ); - return; - } - - let { heartbeat } = request; - const lifetime = getLifetime(request); - delete request.lifetime; - delete request.heartbeat; - - endOfLife[id] = Date.now() + lifetime; - - async function lifetimeLoop() { - while (!done) { - await delay(7500); - if (!endOfLife[id] || endOfLife[id] <= Date.now()) { - end(); - return; - } - } - } - lifetimeLoop(); - - async function heartbeatLoop() { - let hb = parseFloat(heartbeat); - if (hb < MIN_HEARTBEAT) { - hb = MIN_HEARTBEAT; - } else if (hb > MAX_HEARTBEAT) { - hb = MAX_HEARTBEAT; - } - await delay(hb); - while (!done) { - const timeSinceLast = Date.now() - lastSend; - if (timeSinceLast < hb) { - // no neeed to send heartbeat yet - await delay(hb - timeSinceLast); - continue; - } - respond(undefined, ""); - await delay(hb); - } - } - - try { - if (!isValidUUID(account_id)) { - throw Error("account_id must be a valid uuid"); - } - // send the id first - respond(undefined, { id, lifetime }); - db().user_query({ - ...request, - account_id, - changes: id, - cb: respond, - }); - - if (heartbeat) { - heartbeatLoop(); - } - } catch (err) { - if (!done) { - respond(`${err}`); - } - } -} diff --git a/src/packages/nats/files/util.ts b/src/packages/nats/files/util.ts deleted file mode 100644 index 8e46415134..0000000000 --- a/src/packages/nats/files/util.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { delay } from "awaiting"; -import { waitUntilConnected } from "@cocalc/nats/util"; -import { getLogger } from "@cocalc/nats/client"; - -const logger = getLogger("files:util"); - -export async function runLoop({ subs, listen, opts, subject, nc }) { - while (true) { - const sub = nc.subscribe(subject); - subs[subject] = sub; - try { - await listen({ ...opts, sub }); - } catch (err) { - logger.debug(`runLoop: error - ${err}`); - } - if (subs[subject] == null) return; - await delay(3000 + Math.random()); - await waitUntilConnected(); - if (subs[subject] == null) return; - logger.debug(`runLoop: will restart`); - } -} diff --git a/src/packages/nats/hub-api/projects.ts b/src/packages/nats/hub-api/projects.ts deleted file mode 100644 index a0707b9548..0000000000 --- a/src/packages/nats/hub-api/projects.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { authFirstRequireAccount } from "./util"; -import { type CreateProjectOptions } from "@cocalc/util/db-schema/projects"; -import { type UserCopyOptions } from "@cocalc/util/db-schema/projects"; - -export const projects = { - createProject: authFirstRequireAccount, - copyPathBetweenProjects: authFirstRequireAccount, -}; - -export interface Projects { - // request to have NATS permissions to project subjects. - createProject: (opts: CreateProjectOptions) => Promise; - copyPathBetweenProjects: (opts: UserCopyOptions) => Promise; -} diff --git a/src/packages/nats/primus.ts b/src/packages/nats/primus.ts deleted file mode 100644 index 2bbe5b8997..0000000000 --- a/src/packages/nats/primus.ts +++ /dev/null @@ -1,355 +0,0 @@ -/* -Implement something that acts like a websocket as exposed in Primus, but using NATS. - -Development: - -1. Change to a directly like packages/project that imports nats and backend - -2. Example session: - -~/cocalc/src/packages/project$ node -... - -Primus = require('@cocalc/nats/primus').Primus; -env = await require('@cocalc/backend/nats').getEnv(); -server = new Primus({subject:'test',env,role:'server',id:'s'}); -sparks = []; server.on("connection", (spark) => sparks.push(spark)) -client = new Primus({subject:'test',env,role:'client',id:'c0'}); - -sparks[0] -client.on('data',(data)=>console.log('client got', data));0 -sparks[0].write("foo") - -s9 = server.channel('9') -c9 = client.channel('9') -c9.on("data", (data)=>console.log('c9 got', data));0 -s9.on("data", (data)=>console.log('s9 got', data));0 - -c9.write("from client 9") -s9.write("from the server 9") - -client_b = new Primus({subject:'test',env,role:'client',id:'cb'}); -c9b = client_b.channel('9') -c9b.on("data", (data)=>console.log('c9b got', data));0 - -s9.sparks['cb'].write('blah') - -*/ - -import { EventEmitter } from "events"; -import { type NatsEnv } from "@cocalc/nats/types"; -import { delay } from "awaiting"; -import { encodeBase64 } from "@cocalc/nats/util"; - -export type Role = "client" | "server"; - -const PING_INTERVAL = 10000; - -// function otherRole(role: Role): Role { -// return role == "client" ? "server" : "client"; -// } -interface PrimusOptions { - subject: string; - channelName?: string; - env: NatsEnv; - role: Role; - id: string; -} - -const connections: { [key: string]: Primus } = {}; -export function getPrimusConnection(opts: PrimusOptions): Primus { - const key = getKey(opts); - if (connections[key] == null) { - connections[key] = new Primus(opts); - } - return connections[key]; -} - -function getKey({ subject, channelName, role, id }: PrimusOptions) { - return JSON.stringify({ subject, channelName, role, id }); -} - -function getSubjects({ subject, id, channel }) { - const subjects = { - // control = request/response control channel; clients tell - // server they are connecting via this - control: `${subject}.control`, - // server = a server spark listens on server and client - // publishes to server with their id - server: `${subject}.server.${id}`, - // client = client connection listens on this and - // server spark writes to it - client: `${subject}.client.${id}`, - // channel = when set all clients listen on - // this; server sends to this. - clientChannel: `${subject}.channel.client`, - serverChannel: `${subject}.channel.server`, - }; - if (channel) { - // use base64 encoding so channel can be any string. - const segment = encodeBase64(channel); - for (const k in subjects) { - subjects[k] += `.${segment}`; - } - } - return subjects; -} - -type State = "ready" | "closed"; - -export class Primus extends EventEmitter { - subject: string; - channelName: string; - env: NatsEnv; - role: Role; - id: string; - subscribe: string; - sparks: { [id: string]: Spark } = {}; - subjects: { - control: string; - server: string; - client: string; - clientChannel: string; - serverChannel: string; - }; - // this is just for compat with primus api: - address = { ip: "" }; - conn: { id: string }; - subs: any[] = []; - OPEN = 1; - CLOSE = 0; - readyState: 0; - state: State = "ready"; - - constructor({ subject, channelName = "", env, role, id }: PrimusOptions) { - super(); - - // console.log("PRIMUS Creating", { - // subject, - // id, - // channel: channelName, - // }); - - this.subject = subject; - this.channelName = channelName; - this.env = env; - this.role = role; - this.id = id; - this.conn = { id }; - this.subjects = getSubjects({ - subject, - id, - channel: channelName, - }); - if (role == "server") { - this.serve(); - } else { - this.client(); - } - if (this.channelName) { - this.subscribeChannel(); - } - } - - forEach = (f: (spark, id) => void) => { - for (const id in this.sparks) { - f(this.sparks[id], id); - } - }; - - destroy = () => { - if (this.state == "closed") { - return; - } - this.state = "closed"; - delete connections[getKey(this)]; - for (const sub of this.subs) { - sub.close(); - } - this.subs = []; - for (const id in this.sparks) { - this.sparks[id].destroy(); - } - this.sparks = {}; - }; - - end = () => this.destroy(); - - close = () => this.destroy(); - - connect = () => {}; - - private serve = async () => { - if (this.role != "server") { - throw Error("only server can serve"); - } - this.deleteSparks(); - const sub = this.env.nc.subscribe(this.subjects.control); - this.subs.push(sub); - for await (const mesg of sub) { - const data = this.env.jc.decode(mesg.data) ?? ({} as any); - if (data.cmd == "ping") { - const spark = this.sparks[data.id]; - if (spark != null) { - spark.lastPing = Date.now(); - } - } else if (data.cmd == "connect") { - const spark = new Spark({ - primus: this, - id: data.id, - }); - this.sparks[data.id] = spark; - this.emit("connection", spark); - mesg.respond(this.env.jc.encode({ status: "ok" })); - } - } - }; - - private deleteSparks = async () => { - while (this.state != "closed") { - for (const id in this.sparks) { - const spark = this.sparks[id]; - if (Date.now() - spark.lastPing > PING_INTERVAL * 1.5) { - spark.destroy(); - } - } - await delay(PING_INTERVAL * 1.5); - } - }; - - private client = async () => { - if (this.role != "client") { - throw Error("only client can connect"); - } - const mesg = this.env.jc.encode({ - cmd: "connect", - id: this.id, - }); - console.log("Nats Primus: connecting..."); - await this.env.nc.publish(this.subjects.control, mesg); - this.clientPing(); - console.log("Nats Primus: connected:"); - const sub = this.env.nc.subscribe(this.subjects.client); - this.subs.push(sub); - for await (const mesg of sub) { - const data = this.env.jc.decode(mesg.data) ?? ({} as any); - this.emit("data", data); - } - }; - - private clientPing = async () => { - while (this.state != "closed") { - try { - await this.env.nc.publish( - this.subjects.control, - this.env.jc.encode({ - cmd: "ping", - id: this.id, - }), - ); - } catch { - // if ping fails, connection is not working, so die. - this.destroy(); - return; - } - await delay(PING_INTERVAL); - } - }; - - private subscribeChannel = async () => { - const subject = - this.role == "client" - ? this.subjects.clientChannel - : this.subjects.serverChannel; - const sub = this.env.nc.subscribe(subject); - this.subs.push(sub); - for await (const mesg of sub) { - const data = this.env.jc.decode(mesg.data) ?? ({} as any); - this.emit("data", data); - } - }; - - // client: writes to server - // server: write to ALL connected clients in channel model. - write = (data) => { - let subject; - if (this.role == "server") { - if (!this.channel) { - throw Error("broadcast write not implemented when not in channel mode"); - } - subject = this.subjects.clientChannel; - } else { - subject = this.subjects.server; - } - this.env.nc.publish(subject, this.env.jc.encode(data)); - return true; - }; - - channel = (channelName: string) => { - return getPrimusConnection({ - subject: this.subject, - channelName, - env: this.env, - role: this.role, - id: this.id, - }); - }; -} - -// only used on the server -export class Spark extends EventEmitter { - primus: Primus; - id: string; - subjects; - lastPing = Date.now(); - // this is just for compat with primus api: - address = { ip: "" }; - conn: { id: string }; - subs: any[] = []; - state: State = "ready"; - - constructor({ primus, id }) { - super(); - this.primus = primus; - const { subject, channelName } = primus; - this.id = id; - this.conn = { id }; - this.subjects = getSubjects({ - subject, - id, - channel: channelName, - }); - this.init(); - } - - destroy = () => { - if (this.state == "closed") { - return; - } - this.state = "closed"; - for (const sub of this.subs) { - sub.close(); - } - this.subs = []; - delete this.primus.sparks[this.id]; - }; - - end = () => this.destroy(); - - private init = async () => { - const sub = this.primus.env.nc.subscribe(this.subjects.server); - this.subs.push(sub); - for await (const mesg of sub) { - const data = this.primus.env.jc.decode(mesg.data); - this.emit("data", data); - } - }; - - write = (data) => { - this.primus.env.nc.publish( - this.subjects.client, - this.primus.env.jc.encode(data), - ); - return true; - }; -} diff --git a/src/packages/nats/service/index.ts b/src/packages/nats/service/index.ts deleted file mode 100644 index 8007ade3ab..0000000000 --- a/src/packages/nats/service/index.ts +++ /dev/null @@ -1,8 +0,0 @@ -export type { - ServiceDescription, - CallNatsServiceFunction, - ServiceCall, - CreateNatsServiceFunction, - NatsService, -} from "./service"; -export { callNatsService, createNatsService } from "./service"; diff --git a/src/packages/nats/service/many.ts b/src/packages/nats/service/many.ts deleted file mode 100644 index 2ebc4b5e16..0000000000 --- a/src/packages/nats/service/many.ts +++ /dev/null @@ -1,77 +0,0 @@ -/* -Support requestMany and respond to requestMany request transparently. - -If the request is sent via the function requestMany below, then: - (1) it contains the HEADER ("requestMany") with value "Empty", - (2) it combines all the responses together (until receiving Empty) and returns that - -On the other side, respondMany looks for HEADER and if it is set, -breaks up the response data into maximum size chunks based on the -server configured max payload size. - -By using this pair of functions the client can control whether or not -request many is used for a particular request. In particular, if the -header isn't set to request many, then no extra messages get sent back. -*/ - -import { Empty, headers } from "@nats-io/nats-core"; -import { getMaxPayload } from "@cocalc/nats/util"; - -export async function respondMany({ mesg, data }) { - if (!hasRequestManyHeader(mesg)) { - // console.log("respondMany: using NORMAL response"); - // header not set, so just send a normal response. - await mesg.respond(data); - return; - } - // console.log("respondMany: using CHUNKED response"); - // header set, so send response as multiple messages broken into - // chunks followed by an Empty message to terminate. - const maxPayload = await getMaxPayload(); - for (let i = 0; i < data.length; i += maxPayload) { - const slice = data.slice(i, i + maxPayload); - await mesg.respond(slice); - } - await mesg.respond(Empty); -} - -export async function requestMany({ - nc, - subject, - data, - maxWait, -}: { - nc; - subject: string; - data; - maxWait?: number; -}): Promise<{ data: Buffer }> { - // set a special header so that server knows to use our respond many protocol. - const h = headers(); - h.append(HEADER, TERMINATE); - const v: any[] = []; - for await (const resp of await nc.requestMany(subject, data, { - maxWait, - headers: h, - })) { - if (resp.data.length == 0) { - break; - } - v.push(resp); - } - const respData = Buffer.concat(v.map((x) => x.data)); - return { data: respData }; -} - -export const HEADER = "requestMany"; -// terminate on empty message -- only protocol we support right now -export const TERMINATE = "Empty"; - -function hasRequestManyHeader(mesg) { - for (const [key, value] of mesg.headers ?? []) { - if (key == HEADER && value == TERMINATE) { - return true; - } - } - return false; -} diff --git a/src/packages/nats/service/service.ts b/src/packages/nats/service/service.ts deleted file mode 100644 index 5b279a9204..0000000000 --- a/src/packages/nats/service/service.ts +++ /dev/null @@ -1,481 +0,0 @@ -/* -Simple to use UI to connect anything in cocalc via request/reply services. - -- callNatsService -- createNatsService - -The input is basically where the service is (account, project, public), -and either what message to send or how to handle messages. -Also if the handler throws an error, the caller will throw -an error too. -*/ - -import { - Svcm, - type ServiceInfo, - type ServiceStats, - type ServiceIdentity, -} from "@nats-io/services"; -import { type Location } from "@cocalc/nats/types"; -import { trunc_middle } from "@cocalc/util/misc"; -import { getEnv, getLogger } from "@cocalc/nats/client"; -import { randomId } from "@cocalc/nats/names"; -import { delay } from "awaiting"; -import { EventEmitter } from "events"; -import { requestMany, respondMany } from "./many"; -import { encodeBase64, waitUntilConnected } from "@cocalc/nats/util"; - -const DEFAULT_TIMEOUT = 10 * 1000; -const MONITOR_INTERVAL = 45 * 1000; - -// switching this is awkward since it would have to be changed in projects -// and frontends or things would hang. I'm making it toggleable just for -// dev purposes so we can benchmark. -// Using the service framework gives us no real gain and cost a massive amount -// in terms of subscriptions -- basically there's a whole bunch for every file, etc. -// **In short: Do NOT enable this by default.** -const ENABLE_SERVICE_FRAMEWORK = false; - -const logger = getLogger("nats:service"); - -export interface ServiceDescription extends Location { - service: string; - - description?: string; - - // if true and multiple servers are setup in same "location", then they ALL get to respond (sender gets first response). - all?: boolean; - - // DEFAULT: ENABLE_SERVICE_FRAMEWORK - enableServiceFramework?: boolean; - - subject?: string; -} - -export interface ServiceCall extends ServiceDescription { - mesg: any; - timeout?: number; - - // if true, call returns the raw response message, with no decoding or error wrapping. - // (do not combine with many:true) - raw?: boolean; - - // if true, uses requestMany so **responses can be arbitrarily large**. - // This MUST be set for both client and server! Don't use this unless - // you need it, since every response involves 2 messages instead of 1 - // (the extra termination message). A good example that uses this is - // the jupyter api, since large output gets returned when you click on - // "Fetch more output". - many?: boolean; - - // if it fails with NatsError, we wait for service to be ready and try again, - // unless this is set -- e.g., when waiting for the service in the first - // place we set this to avoid an infinite loop. - noRetry?: boolean; -} - -export async function callNatsService(opts: ServiceCall): Promise { - // console.log("callNatsService", opts); - const env = await getEnv(); - const { nc, jc } = env; - const subject = serviceSubject(opts); - let resp; - const timeout = opts.timeout ?? DEFAULT_TIMEOUT; - const data = jc.encode(opts.mesg); - - const doRequest = async () => { - await waitUntilConnected(); - if (opts.many) { - resp = await requestMany({ nc, subject, data, maxWait: timeout }); - } else { - resp = await nc.request(subject, data, { - timeout, - }); - } - if (opts.raw) { - return resp; - } - const result = jc.decode(resp.data); - if (result?.error) { - throw Error(result.error); - } - return result; - }; - - // we just try to call the service first - try { - return await doRequest(); - } catch (err) { - //console.log(`request to '${subject}' failed -- ${err}`); - // it failed. - if (err.name == "NatsError" && !opts.noRetry) { - // it's a nats problem - const p = opts.path ? `${trunc_middle(opts.path, 64)}:` : ""; - if (err.code == "503") { - // it's actually just not ready, so - // wait for the service to be ready, then try again - await waitForNatsService({ options: opts, maxWait: timeout }); - try { - return await doRequest(); - } catch (err) { - if (err.code == "503") { - err.message = `Not Available: service ${p}${opts.service} is not available`; - } - throw err; - } - } else if (err.code == "TIMEOUT") { - throw Error( - `Timeout: service ${p}${opts.service} did not respond for ${Math.round(timeout / 1000)} seconds`, - ); - } - } - throw err; - } -} - -export type CallNatsServiceFunction = typeof callNatsService; - -export interface Options extends ServiceDescription { - description?: string; - version?: string; - handler: (mesg) => Promise; - // see corresponding call option above. - many?: boolean; -} - -export function createNatsService(options: Options) { - return new NatsService(options); -} - -export type CreateNatsServiceFunction = typeof createNatsService; - -export function serviceSubject({ - service, - - account_id, - browser_id, - - project_id, - compute_server_id, - - path, - - subject, -}: ServiceDescription): string { - if (subject) { - return subject; - } - let segments; - path = path ? encodeBase64(path) : "_"; - if (!project_id && !account_id) { - segments = ["public", service]; - } else if (account_id) { - segments = [ - "services", - `account-${account_id}`, - browser_id ?? "_", - project_id ?? "_", - path ?? "_", - service, - ]; - } else if (project_id) { - segments = [ - "services", - `project-${project_id}`, - compute_server_id ?? "_", - service, - path, - ]; - } - return segments.join("."); -} - -export function serviceName({ - service, - - account_id, - browser_id, - - project_id, - compute_server_id, -}: ServiceDescription): string { - let segments; - if (!project_id && !account_id) { - segments = [service]; - } else if (account_id) { - segments = [`account-${account_id}`, browser_id ?? "-", service]; - } else if (project_id) { - segments = [`project-${project_id}`, compute_server_id ?? "-", service]; - } - return segments.join("-"); -} - -export function serviceDescription({ - description, - path, -}: ServiceDescription): string { - return [description, path ? `\nPath: ${path}` : ""].join(""); -} - -export class NatsService extends EventEmitter { - private options: Options; - private subject: string; - private api?; - private name: string; - - constructor(options: Options) { - super(); - this.options = options; - this.name = serviceName(this.options); - this.subject = serviceSubject(options); - this.startMonitor(); - this.startMainLoop(); - } - - private log = (...args) => { - logger.debug(`service:'${this.name}' -- `, ...args); - }; - - private startMainLoop = async () => { - while (this.subject) { - await this.runService(); - await delay(5000); - } - }; - - // The service monitor checks every MONITOR_INTERVAL when - // connected that the service is definitely working and - // responding to pings. If not, it calls restartService. - private startMonitor = async () => { - while (this.subject) { - this.log(`serviceMonitor: waiting ${MONITOR_INTERVAL}ms...`); - await delay(MONITOR_INTERVAL); - if (this.subject == null) return; - await waitUntilConnected(); - if (this.subject == null) return; - try { - this.log(`serviceMonitor: ping`); - await callNatsService({ ...this.options, mesg: "ping", timeout: 7500 }); - if (this.subject == null) return; - this.log("serviceMonitor: ping SUCCESS"); - } catch (err) { - if (this.subject == null) return; - this.log(`serviceMonitor: ping FAILED -- ${err}`); - this.restartService(); - } - } - }; - - private restartService = () => { - if (this.api) { - this.api.stop(); - delete this.api; - } - this.runService(); - }; - - // create and run the service until something goes wrong, when this - // willl return. It does not throw an error. - private runService = async () => { - try { - await waitUntilConnected(); - this.emit("starting"); - - this.log("starting service"); - const env = await getEnv(); - - // close any subscriptions by this client to the subject, which might be left from previous runs of this service. - // @ts-ignore - for (const sub of env.nc.protocol.subscriptions.subs) { - if (sub[1].subject == this.subject) { - sub[1].close(); - } - } - - const queue = this.options.all ? randomId() : "0"; - if (this.options.enableServiceFramework ?? ENABLE_SERVICE_FRAMEWORK) { - const svcm = new Svcm(env.nc); - const service = await svcm.add({ - name: this.name, - version: this.options.version ?? "0.0.1", - description: serviceDescription(this.options), - queue, - }); - if (!this.subject) { - return; - } - this.api = service.addEndpoint("api", { subject: this.subject }); - } else { - this.api = env.nc.subscribe(this.subject, { queue }); - } - this.emit("running"); - await this.listen(); - } catch (err) { - this.log(`service stopping due to ${err}`); - } - }; - - private listen = async () => { - const env = await getEnv(); - const jc = env.jc; - for await (const mesg of this.api) { - const request = jc.decode(mesg.data) ?? ({} as any); - - // console.logger.debug("handle nats service call", request); - let resp; - if (request == "ping") { - resp = "pong"; - } else { - try { - resp = await this.options.handler(request); - } catch (err) { - resp = { error: `${err}` }; - } - } - try { - const data = jc.encode(resp); - if (this.options.many) { - await respondMany({ mesg, data }); - } else { - await mesg.respond(data); - } - } catch (err) { - // If, e.g., resp is too big, then the error would be - // "NatsError: MAX_PAYLOAD_EXCEEDED" - // and it is of course very important to make the caller aware that - // there was an error, as opposed to just silently leaving - // them hanging forever. - const data = jc.encode({ error: `${err}` }); - if (this.options.many) { - await respondMany({ mesg, data }); - } else { - await mesg.respond(data); - } - } - } - }; - - close = () => { - if (!this.subject) { - return; - } - this.emit("close"); - this.removeAllListeners(); - this.api?.stop(); - delete this.api; - // @ts-ignore - delete this.subject; - // @ts-ignore - delete this.options; - }; -} - -interface ServiceClientOpts { - options: ServiceDescription; - maxWait?: number; - id?: string; -} - -export async function pingNatsService({ - options, - maxWait = 500, - id, -}: ServiceClientOpts): Promise<(ServiceIdentity | string)[]> { - if (!(options.enableServiceFramework ?? ENABLE_SERVICE_FRAMEWORK)) { - // console.log( - // `pingNatsService: ${options.service}.${options.description ?? ""} -- using fallback ping`, - // ); - const pong = await callNatsService({ - ...options, - mesg: "ping", - timeout: Math.max(3000, maxWait), - // set no-retry to avoid infinite loop - noRetry: true, - }); - // console.log( - // `pingNatsService: ${options.service}.${options.description ?? ""} -- success`, - // ); - return [pong]; - } - const env = await getEnv(); - const svc = new Svcm(env.nc); - const m = svc.client({ maxWait, strategy: "stall" }); - const v: ServiceIdentity[] = []; - for await (const ping of await m.ping(serviceName(options), id)) { - v.push(ping); - } - return v; -} - -export async function natsServiceInfo({ - options, - maxWait = 500, - id, -}: ServiceClientOpts): Promise { - if (!(options.enableServiceFramework ?? ENABLE_SERVICE_FRAMEWORK)) { - throw Error(`service framework not enabled for ${options.service}`); - } - const env = await getEnv(); - const svc = new Svcm(env.nc); - const m = svc.client({ maxWait, strategy: "stall" }); - const v: ServiceInfo[] = []; - for await (const info of await m.info(serviceName(options), id)) { - v.push(info); - } - return v; -} - -export async function natsServiceStats({ - options, - maxWait = 500, - id, -}: ServiceClientOpts): Promise { - if (!(options.enableServiceFramework ?? ENABLE_SERVICE_FRAMEWORK)) { - throw Error(`service framework not enabled for ${options.service}`); - } - const env = await getEnv(); - const svc = new Svcm(env.nc); - const m = svc.client({ maxWait, strategy: "stall" }); - const v: ServiceStats[] = []; - for await (const stats of await m.stats(serviceName(options), id)) { - v.push(stats); - } - return v; -} - -export async function waitForNatsService({ - options, - maxWait = 60000, -}: { - options: ServiceDescription; - maxWait?: number; -}) { - let d = 1000; - let m = 100; - const start = Date.now(); - const getPing = async (m: number) => { - try { - await waitUntilConnected(); - return await pingNatsService({ options, maxWait: m }); - } catch { - // ping can fail, e.g, if not connected to nats at all or the ping - // service isn't up yet. - return [] as ServiceIdentity[]; - } - }; - let ping = await getPing(m); - while (ping.length == 0) { - d = Math.min(10000, d * 1.3); - m = Math.min(1500, m * 1.3); - if (Date.now() - start + d >= maxWait) { - logger.debug( - `timeout waiting for ${serviceName(options)} to start...`, - d, - ); - throw Error("timeout"); - } - await delay(d); - ping = await getPing(m); - } - return ping; -} diff --git a/src/packages/nats/sync/akv.ts b/src/packages/nats/sync/akv.ts deleted file mode 100644 index a5276b9b7a..0000000000 --- a/src/packages/nats/sync/akv.ts +++ /dev/null @@ -1,173 +0,0 @@ -/* -Asynchronous Memory Efficient Access to Key:Value Store - -This provides the same abstraction as dkv, except it doesn't download any -data to the client until you actually call get. The calls to get and -set are thus async. - -Because AsyncKV has no global knowledge of this key:value store, the inventory -is not updated and limits are not enforced. Of course chunking (storing large -values properly) is supported. - -There is no need to close this because it is stateless. - -DEVELOPMENT: - -~/cocalc/src/packages/backend$ node -> t = require("@cocalc/backend/nats/sync").akv({name:'test'}) - -*/ - -import { GeneralKV } from "./general-kv"; -import { getEnv } from "@cocalc/nats/client"; -import { type DKVOptions, getPrefix } from "./dkv"; -import { once } from "@cocalc/util/async-utils"; -import { jsName } from "@cocalc/nats/names"; -import { encodeBase64 } from "@cocalc/nats/util"; - -export class AKV { - private options: DKVOptions; - private prefix: string; - private noChunks?: boolean; - - constructor({ noChunks, ...options }: DKVOptions & { noChunks?: boolean }) { - this.options = options; - this.noChunks = noChunks; - const { name, valueType = "json" } = options; - this.prefix = getPrefix({ - name, - valueType, - options, - }); - } - - private encodeKey = (key) => { - if (typeof key != "string") { - key = `${key}`; - } - return key ? `${this.prefix}.${encodeBase64(key)}` : this.prefix; - }; - - private getGeneralKVForOneKey = async ( - key: string, - { noWatch = true }: { noWatch?: boolean } = {}, - ): Promise> => { - const { valueType = "json", limits, account_id, project_id } = this.options; - const filter = this.encodeKey(key); - const kv = new GeneralKV({ - name: jsName({ account_id, project_id }), - env: await getEnv(), - // IMPORTANT: need both filter and .> to get CHUNKS in case of chunked data! - filter: [filter, filter + ".>"], - limits, - valueType, - noWatch, - noGet: noWatch && this.noChunks, - }); - await kv.init(); - return kv; - }; - - // Just get one value asynchronously, rather than the entire dkv. - // If the timeout option is given and the value of key is not set, - // will wait until that many ms for the key to get - get = async (key: string, { timeout }: { timeout?: number } = {}) => { - const start = Date.now(); - let noWatch = true; - if (timeout) { - // there's a timeout so in this unusual nondefault case we will watch: - noWatch = false; - } - const kv = await this.getGeneralKVForOneKey(key, { noWatch }); - const filter = this.encodeKey(key); - if (noWatch && this.noChunks) { - const x = await kv.getDirect(filter); - await kv.close(); - return x; - } - try { - let value = kv.get(filter); - if (!timeout) { - return value; - } - while (value === undefined && Date.now() - start <= timeout) { - try { - await once(kv, "change", timeout - (Date.now() - start)); - } catch { - // failed due to timeout -- result is undefined since key isn't set - return undefined; - } - value = kv.get(filter); - } - return value; - } finally { - await kv.close(); - } - }; - - headers = async (key: string) => { - const kv = await this.getGeneralKVForOneKey(key); - const filter = this.encodeKey(key); - if (this.noChunks) { - const x = await kv.getDirect(filter); - if (x === undefined) { - return; - } - } - const h = kv.headers(filter); - await kv.close(); - return h; - }; - - time = async (key: string) => { - const kv = await this.getGeneralKVForOneKey(key); - const filter = this.encodeKey(key); - if (this.noChunks) { - const x = await kv.getDirect(filter); - if (x === undefined) { - return; - } - } - const t = kv.time(filter); - await kv.close(); - return t; - }; - - delete = async (key: string) => { - const kv = await this.getGeneralKVForOneKey(key); - const filter = this.encodeKey(key); - await kv.delete(filter); - await kv.close(); - }; - - // NOTE: set does NOT update the inventory or apply limits, since this - // has no global knowledge of the kv. - set = async ( - key: string, - value: T, - options?: { headers?: { [key: string]: string }; previousSeq?: number }, - ) => { - const kv = await this.getGeneralKVForOneKey(key); - const filter = this.encodeKey(key); - await kv.set(filter, value, { - ...options, - headers: { ...options?.headers }, - }); - }; - - seq = async (key: string) => { - const kv = await this.getGeneralKVForOneKey(key); - const filter = this.encodeKey(key); - if (this.noChunks) { - const x = await kv.getDirect(filter); - if (x === undefined) { - return; - } - } - return kv.seq(filter); - }; -} - -export function akv(opts: DKVOptions) { - return new AKV(opts); -} diff --git a/src/packages/nats/sync/dkv.ts b/src/packages/nats/sync/dkv.ts deleted file mode 100644 index bfc12ba104..0000000000 --- a/src/packages/nats/sync/dkv.ts +++ /dev/null @@ -1,504 +0,0 @@ -/* -Eventually Consistent Distributed Key Value Store - -Supports: - - - automatic value chunking to store arbitrarily large values - - type checking the values - - arbitrary name - - arbitrary keys - - limits - -This downloads ALL data for the key:value store to the client, -then keeps it is in sync with NATS. - -For an alternative space efficient async interface to the EXACT SAME DATA, -see akv.ts. - -IMPORTANT: When you set data in a dkv, it is NOT guaranteed to be saved remotely -forever immediately, obviously. The dkv attempts to save to NATS in the background. -**There are reasons whey saving might fail or data you have set could disappear!** -E.g., if you set the max_msg_size limit, and try to set a value that is too -large, then it is removed during save and a 'reject' event is fired. -The other limits will silently delete data for other reasons as well (e.g., too old, -too many messages). - -EVENTS: - -- 'change', {key:string, value?:T, prev:T} -- there is a change. - if value===undefined, that means that key is deleted and the value used to be prev. - -- 'reject', {key, value} -- data you set is rejected when trying to save, e.g., if too large - -- 'stable' -- there are no unsaved changes and all saved changes have been - echoed back from server. - -- 'closed' -- the dkv is now closed. Note that close's are reference counted, e.g., you can - grab the same dkv in multiple places in your code, close it when do with each, and it - is freed when the number of closes equals the number of objects you created. - -Merge conflicts are handled by your custom merge function, and no event is fired. - -DEVELOPMENT: - -From node.js - - ~/cocalc/src/packages/backend$ n - Welcome to Node.js v18.17.1. - Type ".help" for more information. - > t = await require("@cocalc/backend/nats/sync").dkv({name:'test'}) - -From the browser: - -If you want a persistent distributed key:value store in the browser, -which shares state to all browser clients for a given **account_id**, -do this in the dev console: - - > a = await cc.client.nats_client.dkv({name:'test', account_id:cc.client.account_id}) - -Then do the same thing in another dev console in another browser window: - - > a = await cc.client.nats_client.dkv({name:'test', account_id:cc.client.account_id}) - -Now do this in one: - - > a.x = 10 - -and - - > a.x - 10 - -in the other. Yes, it's that easy to have a persistent distributed eventually consistent -synchronized key-value store! - -For library code, replace cc.client by webapp_client, which you get via: - - import { webapp_client } from "@cocalc/frontend/webapp-client" - -If instead you need to share state with a project (or compute server), use - -> b = await cc.client.nats_client.dkv({name:'test', project_id:'...'}) - - -UNIT TESTS: See backend/nats/test/ - -They aren't right here, because this module doesn't have info to connect to NATS. - -*/ - -import { EventEmitter } from "events"; -import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; -import { GeneralDKV, TOMBSTONE, type MergeFunction } from "./general-dkv"; -import { jsName } from "@cocalc/nats/names"; -import { userKvKey, type KVOptions } from "./kv"; -import { localLocationName } from "@cocalc/nats/names"; -import refCache from "@cocalc/util/refcache"; -import { getEnv } from "@cocalc/nats/client"; -import { - inventory, - INVENTORY_NAME, - THROTTLE_MS, - type Inventory, -} from "./inventory"; -import { asyncThrottle } from "@cocalc/util/async-utils"; -import { delay } from "awaiting"; -import { decodeBase64, encodeBase64 } from "@cocalc/nats/util"; -import { getLogger } from "@cocalc/nats/client"; -import { waitUntilConnected } from "@cocalc/nats/util"; - -const logger = getLogger("dkv"); - -export interface DKVOptions extends KVOptions { - merge?: MergeFunction; - noAutosave?: boolean; - noInventory?: boolean; -} - -export class DKV extends EventEmitter { - generalDKV?: GeneralDKV; - name: string; - private prefix: string; - private opts; - private keys: { [encodedKey: string]: string } = {}; - - constructor(options: DKVOptions) { - super(); - const { - name, - account_id, - project_id, - merge, - env, - noAutosave, - limits, - noInventory, - desc, - valueType = "json", - } = options; - if (env == null) { - throw Error("env must not be null"); - } - if ( - noInventory || - (process.env.COCALC_TEST_MODE && noInventory == null) || - name == INVENTORY_NAME - ) { - // @ts-ignore - this.updateInventory = () => {}; - } - // name of the jetstream key:value store. - this.name = name; - - this.prefix = getPrefix({ name, valueType, options }); - - this.opts = { - location: { account_id, project_id }, - name: jsName({ account_id, project_id }), - desc, - noInventory, - filter: [this.prefix, `${this.prefix}.>`], - env, - merge, - noAutosave, - limits, - valueType, - }; - - this.init(); - return new Proxy(this, { - deleteProperty(target, prop) { - if (typeof prop == "string") { - target.delete(prop); - } - return true; - }, - set(target, prop, value) { - prop = String(prop); - if (prop == "_eventsCount" || prop == "_events" || prop == "close") { - target[prop] = value; - return true; - } - if (target[prop] != null) { - throw Error(`method name '${prop}' is read only`); - } - target.set(prop, value); - return true; - }, - get(target, prop) { - return target[String(prop)] ?? target.get(String(prop)); - }, - }); - } - - init = reuseInFlight(async () => { - if (this.generalDKV != null) { - return; - } - // the merge conflict algorithm must be adapted since we encode - // the key in the header. - const merge = (opts) => { - // here is what the input might look like: - // opts = { - // key: '71d7616250fed4dc27b70ee3b934178a3b196bbb.11f6ad8ec52a2984abaafd7c3b516503785c2072', - // remote: { key: 'x', value: 10 }, - // local: { key: 'x', value: 5 }, - // prev: { key: 'x', value: 3 } - // } - const key = this.getKey(opts.key); - if (key == null) { - console.warn("BUG in merge conflict resolution", opts); - throw Error("local key must be defined"); - } - const { local, remote, prev } = opts; - try { - return this.opts.merge?.({ key, local, remote, prev }) ?? local; - } catch (err) { - console.warn("exception in merge conflict resolution", err); - return local; - } - }; - this.generalDKV = new GeneralDKV({ - ...this.opts, - merge, - desc: `${this.name} ${this.opts.desc ?? ""}`, - }); - this.generalDKV.on("change", ({ key, value, prev }) => { - if (this.generalDKV == null) { - return; - } - let decodedKey; - try { - decodedKey = this.getKey(key); - } catch (err) { - // key is missing so at this point there is no knowledge of it and - // nothing we can alert on. - // TODO: may remove this when/if we completely understand why - // this ever happens - // console.log("WARNING: missing key for -- ", { key, err }); - return; - } - if (value !== undefined && value !== TOMBSTONE) { - this.emit("change", { - key: decodedKey, - value, - prev, - }); - } else { - // value is undefined or TOMBSTONE, so it's a delete, so do not set value here - this.emit("change", { key: decodedKey, prev }); - } - }); - this.generalDKV.on("reject", ({ key, value }) => { - if (this.generalDKV == null) { - return; - } - if (value != null) { - this.emit("reject", { key: this.getKey(key), value }); - } - }); - this.generalDKV.on("stable", () => this.emit("stable")); - await this.generalDKV.init(); - this.updateInventory(); - }); - - close = async () => { - const generalDKV = this.generalDKV; - if (generalDKV == null) { - return; - } - delete this.generalDKV; - await generalDKV.close(); - // @ts-ignore - delete this.opts; - this.emit("closed"); - this.removeAllListeners(); - }; - - delete = (key: string) => { - if (this.generalDKV == null) { - throw Error("closed"); - } - this.generalDKV.delete(this.encodeKey(key)); - this.updateInventory(); - }; - - clear = () => { - if (this.generalDKV == null) { - throw Error("closed"); - } - this.generalDKV.clear(); - this.updateInventory(); - }; - - // server assigned time - time = (key?: string): Date | undefined | { [key: string]: Date } => { - if (this.generalDKV == null) { - throw Error("closed"); - } - const times = this.generalDKV.time(key ? this.encodeKey(key) : undefined); - if (key != null || times == null) { - return times; - } - const obj = this.generalDKV.getAll(); - const x: any = {}; - for (const k in obj) { - const { key } = obj[k]; - x[key] = times[k]; - } - return x; - }; - - // WARNING: (1) DO NOT CHANGE THIS or all stored data will become invalid. - // (2) This definition is used implicitly in akv.ts also! - // The encoded key which we actually store in NATS. It has to have - // a restricted form, and a specified prefix, which - // is why the hashing, etc. This allows arbitrary keys. - // We have to monkey patch nats to accept even base64 keys!) - // There are NOT issues with key length though. This same strategy of encoding - // keys using base64 is used by Nats object store: - // https://github.com/nats-io/nats-architecture-and-design/blob/main/adr/ADR-20.md#object-name - private encodeKey = (key) => { - if (typeof key != "string") { - key = `${key}`; - } - return key ? `${this.prefix}.${encodeBase64(key)}` : this.prefix; - }; - - // decodeKey is the inverse of encodeKey - private decodeKey = (encodedKey) => { - const v = encodedKey.split("."); - return v[1] ? decodeBase64(v[1]) : ""; - }; - - has = (key: string): boolean => { - if (this.generalDKV == null) { - throw Error("closed"); - } - return this.generalDKV.has(this.encodeKey(key)); - }; - - get = (key: string): T | undefined => { - if (this.generalDKV == null) { - throw Error("closed"); - } - return this.generalDKV.get(this.encodeKey(key)); - }; - - getAll = (): { [key: string]: T } => { - if (this.generalDKV == null) { - throw Error("closed"); - } - const obj = this.generalDKV.getAll(); - const x: any = {}; - for (const k in obj) { - const key = this.getKey(k); - x[key] = obj[k]; - } - return x; - }; - - private getKey = (k) => { - if (this.keys[k] != null) { - return this.keys[k]; - } - return this.decodeKey(k); - }; - - headers = (key: string) => { - return this.generalDKV?.headers(this.encodeKey(key)); - }; - - get length(): number { - if (this.generalDKV == null) { - throw Error("closed"); - } - return this.generalDKV.length; - } - - set = ( - key: string, - value: T, - // NOTE: if you call this.headers(n) it is NOT visible until the publish is confirmed. - // This could be changed with more work if it matters. - options?: { headers?: { [key: string]: string } }, - ): void => { - if (this.generalDKV == null) { - throw Error("closed"); - } - if (value === undefined) { - // undefined can't be JSON encoded, so we can't possibly represent it, and this - // *must* be treated as a delete. - // NOTE that jc.encode encodes null and undefined the same, so supporting this - // as a value is just begging for misery. - this.delete(key); - this.updateInventory(); - return; - } - const encodedKey = this.encodeKey(key); - this.keys[encodedKey] = key; - this.generalDKV.set(encodedKey, value, { - headers: { ...options?.headers }, - }); - this.updateInventory(); - }; - - hasUnsavedChanges = (): boolean => { - if (this.generalDKV == null) { - return false; - } - return this.generalDKV.hasUnsavedChanges(); - }; - - unsavedChanges = (): string[] => { - const generalDKV = this.generalDKV; - if (generalDKV == null) { - return []; - } - return generalDKV.unsavedChanges().map((key) => this.getKey(key)); - }; - - isStable = () => { - return this.generalDKV?.isStable(); - }; - - save = async () => { - return await this.generalDKV?.save(); - }; - - private updateInventory = asyncThrottle( - async () => { - if (this.generalDKV == null || this.opts.noInventory) { - return; - } - await delay(500); - if (this.generalDKV == null) { - return; - } - const { valueType } = this.opts; - const name = this.name; - let inv: null | Inventory = null; - - try { - await waitUntilConnected(); - inv = await inventory(this.opts.location); - if (this.generalDKV == null) { - return; - } - if (!inv.needsUpdate({ name, type: "kv", valueType })) { - return; - } - const stats = this.generalDKV.stats(); - if (stats == null) { - return; - } - const { count, bytes } = stats; - inv.set({ - type: "kv", - name, - count, - bytes, - desc: this.opts.desc, - valueType: this.opts.valueType, - limits: this.opts.limits, - }); - } catch (err) { - logger.debug( - `WARNING: unable to update inventory for ${this.opts?.name} -- ${err}`, - ); - } finally { - await inv?.close(); - } - }, - THROTTLE_MS, - { leading: true, trailing: true }, - ); -} - -// *** WARNING: THIS CAN NEVER BE CHANGE! ** -// The recipe for 'this.prefix' must never be changed, because -// it determines where the data is actually stored. If you change -// it, then every user's data vanishes. -export function getPrefix({ name, valueType, options }) { - return encodeBase64( - JSON.stringify([name, valueType, localLocationName(options)]), - ); -} - -export const cache = refCache({ - name: "dkv", - createKey: userKvKey, - createObject: async (opts) => { - await waitUntilConnected(); - if (opts.env == null) { - opts.env = await getEnv(); - } - const k = new DKV(opts); - await k.init(); - return k; - }, -}); - -export async function dkv(options: DKVOptions): Promise> { - return await cache(options); -} diff --git a/src/packages/nats/sync/dstream.ts b/src/packages/nats/sync/dstream.ts deleted file mode 100644 index ef38a7a937..0000000000 --- a/src/packages/nats/sync/dstream.ts +++ /dev/null @@ -1,491 +0,0 @@ -/* -Eventually Consistent Distributed Event Stream - -DEVELOPMENT: - - -# in node -- note the package directory!! -~/cocalc/src/packages/backend n -Welcome to Node.js v18.17.1. -Type ".help" for more information. - -> s = await require("@cocalc/backend/nats/sync").dstream({name:'test'}); - - -> s = await require("@cocalc/backend/nats/sync").dstream({project_id:cc.current().project_id,name:'foo'});0 - - -See the guide for dkv, since it's very similar, especially for use in a browser. - -*/ - -import { EventEmitter } from "events"; -import { - Stream, - type StreamOptions, - type UserStreamOptions, - userStreamOptionsKey, - last, -} from "./stream"; -import { EphemeralStream } from "./ephemeral-stream"; -import { jsName, streamSubject, randomId } from "@cocalc/nats/names"; -import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; -import { delay } from "awaiting"; -import { map as awaitMap } from "awaiting"; -import { isNumericString } from "@cocalc/util/misc"; -import refCache from "@cocalc/util/refcache"; -import { type JsMsg } from "@nats-io/jetstream"; -import { getEnv } from "@cocalc/nats/client"; -import { inventory, THROTTLE_MS, type Inventory } from "./inventory"; -import { asyncThrottle } from "@cocalc/util/async-utils"; -import { getClient, type ClientWithState } from "@cocalc/nats/client"; -import { encodeBase64 } from "@cocalc/nats/util"; -import { getLogger } from "@cocalc/nats/client"; -import { waitUntilConnected } from "@cocalc/nats/util"; -import { type Msg } from "@nats-io/nats-core"; -import { headersFromRawMessages } from "./stream"; -import { COCALC_MESSAGE_ID_HEADER } from "./ephemeral-stream"; - -const logger = getLogger("dstream"); - -const MAX_PARALLEL = 250; - -export interface DStreamOptions extends StreamOptions { - noAutosave?: boolean; - noInventory?: boolean; - ephemeral?: boolean; - leader?: boolean; -} - -export class DStream extends EventEmitter { - public readonly name: string; - private stream?: Stream | EphemeralStream; - private messages: T[]; - private raw: (JsMsg | Msg)[][]; - private noAutosave: boolean; - // TODO: using Map for these will be better because we use .length a bunch, which is O(n) instead of O(1). - private local: { [id: string]: T } = {}; - private publishOptions: { - [id: string]: { headers?: { [key: string]: string } }; - } = {}; - private saved: { [seq: number]: T } = {}; - private opts; - private client?: ClientWithState; - - constructor(opts: DStreamOptions) { - super(); - - if ( - opts.noInventory || - opts.ephemeral || - (process.env.COCALC_TEST_MODE && opts.noInventory == null) - ) { - // @ts-ignore - this.updateInventory = () => {}; - } - this.opts = opts; - this.noAutosave = !!opts.noAutosave; - this.name = opts.name; - this.stream = opts.ephemeral ? new EphemeralStream(opts) : new Stream(opts); - this.messages = this.stream.messages; - this.raw = this.stream.raw; - if (!this.noAutosave) { - this.client = getClient(); - this.client.on("connected", this.save); - } - return new Proxy(this, { - get(target, prop) { - return typeof prop == "string" && isNumericString(prop) - ? target.get(parseInt(prop)) - : target[String(prop)]; - }, - }); - } - - init = reuseInFlight(async () => { - if (this.stream == null) { - throw Error("closed"); - } - this.stream.on("change", (mesg: T, raw: JsMsg[]) => { - delete this.saved[last(raw).seq]; - const headers = headersFromRawMessages(raw); - if (headers?.[COCALC_MESSAGE_ID_HEADER]) { - // this is critical with ephemeral-stream.ts, since otherwise there is a moment - // when the same message is in both this.local *and* this.messages, and you'll - // see it doubled in this.getAll(). I didn't see this ever with - // stream.ts, but maybe it is possible. It probably wouldn't impact any application, - // but still it would be a bug to not do this properly, which is what we do here. - delete this.local[headers[COCALC_MESSAGE_ID_HEADER]]; - } - this.emit("change", mesg); - if (this.isStable()) { - this.emit("stable"); - } - }); - this.stream.on("reset", () => { - this.local = {}; - this.saved = {}; - }); - await this.stream.init(); - this.emit("connected"); - this.updateInventory(); - }); - - isStable = () => { - for (const _ in this.saved) { - return false; - } - for (const _ in this.local) { - return false; - } - return true; - }; - - close = async () => { - if (this.stream == null) { - return; - } - if (!this.noAutosave) { - this.client?.removeListener("connected", this.save); - try { - await this.save(); - } catch { - // [ ] TODO: try localStorage or a file?! - } - } - this.stream.close(); - this.emit("closed"); - this.removeAllListeners(); - delete this.stream; - // @ts-ignore - delete this.local; - // @ts-ignore - delete this.messages; - // @ts-ignore - delete this.raw; - }; - - get = (n?): T | T[] => { - if (this.stream == null) { - throw Error("closed"); - } - if (n == null) { - return this.getAll(); - } else { - if (n < this.messages.length) { - return this.messages[n]; - } - const v = Object.keys(this.saved); - if (n < v.length + this.messages.length) { - return this.saved[n - this.messages.length]; - } - return Object.values(this.local)[n - this.messages.length - v.length]; - } - }; - - getAll = (): T[] => { - if (this.stream == null) { - throw Error("closed"); - } - return [ - ...this.messages, - ...Object.values(this.saved), - ...Object.values(this.local), - ]; - }; - - // sequence number of n-th message - seq = (n: number): number | undefined => { - if (n < this.raw.length) { - return last(this.raw[n])?.seq; - } - const v = Object.keys(this.saved); - if (n < v.length + this.raw.length) { - return parseInt(v[n - this.raw.length]); - } - }; - - time = (n: number): Date | undefined => { - if (this.stream == null) { - throw Error("not initialized"); - } - return this.stream.time(n); - }; - - // all server assigned times of messages in the stream. - times = (): (Date | undefined)[] => { - if (this.stream == null) { - throw Error("not initialized"); - } - return this.stream.times(); - }; - - get length(): number { - return ( - this.messages.length + - Object.keys(this.saved).length + - Object.keys(this.local).length - ); - } - - private toValue = (obj) => { - if (this.stream == null) { - throw Error("not initialized"); - } - if (this.stream.valueType == "binary") { - if (!ArrayBuffer.isView(obj)) { - throw Error("value must be an array buffer"); - } - return obj; - } - return this.opts.env.jc.decode(this.opts.env.jc.encode(obj)); - }; - - publish = ( - mesg: T, - // NOTE: if you call this.headers(n) it is NOT visible until the publish is confirmed. - // This could be changed with more work if it matters. - options?: { headers?: { [key: string]: string } }, - ): void => { - const id = randomId(); - this.local[id] = this.toValue(mesg); - if (options != null) { - this.publishOptions[id] = options; - } - if (!this.noAutosave) { - this.save(); - } - this.updateInventory(); - }; - - headers = (n) => { - if (this.stream == null) { - throw Error("closed"); - } - return this.stream.headers(n); - }; - - push = (...args: T[]) => { - if (this.stream == null) { - throw Error("closed"); - } - for (const mesg of args) { - this.publish(mesg); - } - }; - - hasUnsavedChanges = (): boolean => { - if (this.stream == null) { - return false; - } - return Object.keys(this.local).length > 0; - }; - - unsavedChanges = (): T[] => { - return Object.values(this.local); - }; - - save = reuseInFlight(async () => { - let d = 100; - while (true) { - try { - await this.attemptToSave(); - //console.log("successfully saved"); - } catch (err) { - d = Math.min(10000, d * 1.3) + Math.random() * 100; - await delay(d); - console.warn( - `WARNING stream attemptToSave failed -- ${err}`, - this.name, - ); - } - if (!this.hasUnsavedChanges()) { - return; - } - } - }); - - private attemptToSave = reuseInFlight(async () => { - const f = async (id) => { - if (this.stream == null) { - throw Error("closed"); - } - const mesg = this.local[id]; - try { - // @ts-ignore - const { seq } = await this.stream.publish(mesg, { - ...this.publishOptions[id], - msgID: id, - }); - if ((last(this.raw[this.raw.length - 1])?.seq ?? -1) < seq) { - // it still isn't in this.raw - this.saved[seq] = mesg; - } - delete this.local[id]; - delete this.publishOptions[id]; - } catch (err) { - if (err.code == "REJECT") { - delete this.local[id]; - // err has mesg and subject set. - this.emit("reject", { err, mesg }); - } else { - throw err; - } - } - if (this.isStable()) { - this.emit("stable"); - } - }; - // NOTE: ES6 spec guarantees "String keys are returned in the order - // in which they were added to the object." - const ids = Object.keys(this.local); - await awaitMap(ids, MAX_PARALLEL, f); - }); - - // load older messages starting at start_seq - load = async (opts: { start_seq: number }) => { - if (this.stream == null) { - throw Error("closed"); - } - await this.stream.load(opts); - }; - - // this is not synchronous -- it makes sure everything is saved out, - // then purges the stream stored in nats. - // NOTE: other clients will NOT see the result of a purge (unless they reconnect). - purge = async (opts?) => { - await this.save(); - if (this.stream == null) { - throw Error("not initialized"); - } - await this.stream.purge(opts); - }; - - get start_seq(): number | undefined { - return this.stream?.start_seq; - } - - // returns largest sequence number known to this client. - // not optimized to be super fast. - private getCurSeq = (): number | undefined => { - let s = 0; - if (this.raw.length > 0) { - s = Math.max(s, this.seq(this.raw.length - 1)!); - } - for (const t in this.saved) { - const x = parseInt(t); - if (x > s) { - s = x; - } - } - return s ? s : undefined; - }; - - private updateInventory = asyncThrottle( - async () => { - if (this.stream == null || this.opts.noInventory) { - return; - } - await delay(500); - if (this.stream == null) { - return; - } - const name = this.name; - const { valueType } = this.opts; - let inv: null | Inventory = null; - try { - const curSeq = this.getCurSeq(); - if (!curSeq) { - // we know nothing - return; - } - const { account_id, project_id, desc, limits } = this.opts; - await waitUntilConnected(); - inv = await inventory({ account_id, project_id }); - if (this.stream == null) { - return; - } - if (!inv.needsUpdate({ name, type: "stream", valueType })) { - return; - } - - const cur = inv.get({ type: "stream", name, valueType }); - // last update gave info for everything up to and including seq. - const seq = cur?.seq ?? 0; - if (seq + 1 < (this.start_seq ?? 1)) { - // We know data starting at start_seq, but this is strictly - // too far along the sequence. - throw Error("not enough sequence data to update inventory"); - } - - // [ ] TODO: need to take into account cur.seq in computing stats! - - const stats = this.stream?.stats({ start_seq: seq + 1 }); - if (stats == null) { - return; - } - const { count, bytes } = stats; - - inv.set({ - type: "stream", - name, - valueType, - count: count + (cur?.count ?? 0), - bytes: bytes + (cur?.bytes ?? 0), - desc, - limits, - seq: curSeq, - }); - } catch (err) { - logger.debug( - `WARNING: unable to update inventory. name='${this.opts.name} -- ${err}'`, - ); - } finally { - await inv?.close(); - } - }, - THROTTLE_MS, - { leading: true, trailing: true }, - ); -} - -export const cache = refCache({ - name: "dstream", - createKey: userStreamOptionsKey, - createObject: async (options) => { - await waitUntilConnected(); - if (options.env == null) { - options.env = await getEnv(); - } - const { account_id, project_id, name, valueType = "json" } = options; - const jsname = jsName({ account_id, project_id }); - const subjects = streamSubject({ account_id, project_id }); - - // **CRITICAL:** do NOT change how the filter is computed as a function - // of options unless it is backwards compatible, or all user data - // involving streams will just go poof! - const uniqueFilter = JSON.stringify([name, valueType]); - const filter = subjects.replace(">", encodeBase64(uniqueFilter)); - const dstream = new DStream({ - ...options, - name, - jsname, - subjects, - subject: filter, - filter, - }); - await dstream.init(); - return dstream; - }, -}); - -export async function dstream( - options: UserStreamOptions & { - noAutosave?: boolean; - noInventory?: boolean; - leader?: boolean; - ephemeral?: boolean; - }, -): Promise> { - return await cache(options); -} diff --git a/src/packages/nats/sync/ephemeral-stream.ts b/src/packages/nats/sync/ephemeral-stream.ts deleted file mode 100644 index 24b47e2e02..0000000000 --- a/src/packages/nats/sync/ephemeral-stream.ts +++ /dev/null @@ -1,729 +0,0 @@ -/* -An Ephemeral Stream - -DEVELOPMENT: - -~/cocalc/src/packages/backend$ node - - - require('@cocalc/backend/nats'); a = require('@cocalc/nats/sync/ephemeral-stream'); s = await a.estream({name:'test', leader:true}) - - -Testing two at once (a leader and non-leader): - - require('@cocalc/backend/nats'); s = await require('@cocalc/backend/nats/sync').dstream({ephemeral:true,name:'test', leader:1, noAutosave:true}); t = await require('@cocalc/backend/nats/sync').dstream({ephemeral:true,name:'test', leader:0,noAutosave:true}) - -*/ - -import { - type FilteredStreamLimitOptions, - last, - enforceLimits, - enforceRateLimits, - headersFromRawMessages, -} from "./stream"; -import { type NatsEnv, type ValueType } from "@cocalc/nats/types"; -import { EventEmitter } from "events"; -import { Empty, type Msg, type Subscription } from "@nats-io/nats-core"; -import { isNumericString } from "@cocalc/util/misc"; -import type { JSONValue } from "@cocalc/util/types"; -import { - // getMaxPayload, - waitUntilConnected, - encodeBase64, -} from "@cocalc/nats/util"; -import refCache from "@cocalc/util/refcache"; -import { streamSubject } from "@cocalc/nats/names"; -import { getEnv } from "@cocalc/nats/client"; -import { headers as createHeaders } from "@nats-io/nats-core"; -import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; -import { throttle } from "lodash"; -import { once } from "@cocalc/util/async-utils"; -import { callback, delay } from "awaiting"; -import { randomId } from "@cocalc/nats/names"; - -export interface RawMsg extends Msg { - timestamp: number; - seq: number; - sessionId: string; -} - -export const ENFORCE_LIMITS_THROTTLE_MS = process.env.COCALC_TEST_MODE - ? 100 - : 45000; - -const HEADER_PREFIX = "CoCalc-"; - -const COCALC_SEQUENCE_HEADER = `${HEADER_PREFIX}Seq`; -const COCALC_TIMESTAMP_HEADER = `${HEADER_PREFIX}Timestamp`; -const COCALC_OPTIONS_HEADER = `${HEADER_PREFIX}Options`; -const COCALC_SESSION_ID_HEADER = `${HEADER_PREFIX}Session-Id`; -export const COCALC_MESSAGE_ID_HEADER = `${HEADER_PREFIX}Msg-Id`; - -const PUBLISH_TIMEOUT = 7500; - -const DEFAULT_HEARTBEAT_INTERVAL = 30 * 1000; - -export interface EphemeralStreamOptions { - // what it's called - name: string; - // where it is located - account_id?: string; - project_id?: string; - limits?: Partial; - // only load historic messages starting at the given seq number. - start_seq?: number; - desc?: JSONValue; - valueType?: ValueType; - leader?: boolean; - - noCache?: boolean; - heartbeatInterval?: number; -} - -export class EphemeralStream extends EventEmitter { - public readonly name: string; - private readonly subject: string; - private readonly limits: FilteredStreamLimitOptions; - private _start_seq?: number; - public readonly valueType: ValueType; - // don't do "this.raw=" or "this.messages=" anywhere in this class!!! - public readonly raw: RawMsg[][] = []; - public readonly messages: T[] = []; - private readonly msgIDs = new Set(); - - private env?: NatsEnv; - private sub?: Subscription; - private leader: boolean; - private server?: Subscription; - // seq used by the *leader* only to assign sequence numbers - private seq: number = 1; - private lastHeartbeat: number = 0; - private heartbeatInterval: number; - // lastSeq used by clients to keep track of what they have received; if one - // is skipped they reconnect starting with the last one they didn't miss. - private lastSeq: number = 0; - private sendQueue: { data; options?; seq: number; cb: Function }[] = []; - private bytesSent: { [time: number]: number } = {}; - - private sessionId?: string; - - constructor({ - name, - project_id, - account_id, - limits, - start_seq, - valueType = "json", - leader = false, - heartbeatInterval = DEFAULT_HEARTBEAT_INTERVAL, - }: EphemeralStreamOptions) { - super(); - this.valueType = valueType; - this.heartbeatInterval = heartbeatInterval; - this.name = name; - this.leader = !!leader; - const subjects = streamSubject({ account_id, project_id, ephemeral: true }); - this.subject = subjects.replace(">", encodeBase64(name)); - this._start_seq = start_seq; - this.limits = { - max_msgs: -1, - max_age: 0, - max_bytes: -1, - max_msg_size: -1, - max_bytes_per_second: -1, - max_msgs_per_second: -1, - ...limits, - }; - return new Proxy(this, { - get(target, prop) { - return typeof prop == "string" && isNumericString(prop) - ? target.get(parseInt(prop)) - : target[String(prop)]; - }, - }); - } - - init = async () => { - this.env = await getEnv(); - if (!this.leader) { - // try to get current data from a leader - await this.getAllFromLeader({ - start_seq: this._start_seq ?? 0, - noEmit: true, - }); - } else { - // start listening on the subject for new data - this.serve(); - } - // NOTE: if we miss a message between getAllFromLeader and when we start listening, - // then the sequence number will have a gap, and we'll immediately reconnect, starting - // at the right point. So no data can possibly be lost. - this.listen(); - if (!this.leader) { - this.heartbeatMonitor(); - } - if (this.env?.nc?.on != null) { - this.env.nc.on("reconnect", this.reconnect); - } - }; - - private resetState = () => { - delete this.sessionId; - this.bytesSent = {}; - this.msgIDs.clear(); - this.raw.length = 0; - this.messages.length = 0; - this.seq = 0; - this.sendQueue.length = 0; - this.lastSeq = 0; - delete this._start_seq; - this.emit("reset"); - }; - - private reset = async () => { - this.resetState(); - await this.reconnect(); - }; - - close = () => { - if (this.env?.nc?.removeListener != null) { - this.env.nc.removeListener("reconnect", this.reconnect); - } - delete this.env; - this.removeAllListeners(); - // @ts-ignore - this.sub?.close(); - delete this.sub; - // @ts-ignore - this.server?.close(); - delete this.server; - }; - - private getAllFromLeader = async ({ - maxWait = 30000, - start_seq = 0, - noEmit, - }: { maxWait?: number; start_seq?: number; noEmit?: boolean } = {}) => { - if (this.leader) { - throw Error("this is the leader"); - } - let d = 1000; - while (this.env != null) { - await waitUntilConnected(); - if (this.env == null) { - return; - } - // console.log("getAllFromLeader", { start_seq }); - try { - for await (const raw0 of await this.env.nc.requestMany( - this.subject + ".all", - this.env.jc.encode({ start_seq }), - { maxWait }, - )) { - this.lastHeartbeat = Date.now(); - if (raw0.data.length == 0) { - // done - return; - } - const raw = getRawMsg(raw0); - if ( - !this.leader && - this.sessionId && - this.sessionId != raw.sessionId - ) { - await this.reset(); - return; - } else if (this.lastSeq && raw.seq > this.lastSeq + 1) { - // console.log("skipped a sequence number - reconnecting"); - await this.reconnect(); - return; - } else if (raw.seq <= this.lastSeq) { - // already saw this - continue; - } - if (!this.sessionId) { - this.sessionId = raw.sessionId; - } - this.lastSeq = raw.seq; - const mesg = this.decodeValue(raw.data); - this.messages.push(mesg); - this.raw.push([raw]); - if (!noEmit) { - this.emit("change", mesg, [raw]); - } - } - return; - } catch (err) { - // console.log(`err connecting -- ${err}`); - if (err.code == "503") { - // leader just isn't ready yet - d = Math.min(15000, d * 1.3); - await delay(d); - continue; - } else { - throw err; - } - } - } - }; - - private serve = async () => { - if (this.env == null) { - throw Error("closed"); - } - this.sessionId = randomId(); - this.sendHeartbeats(); - this.server = this.env.nc.subscribe(this.subject + ".>"); - for await (const raw of this.server) { - if (raw.subject.endsWith(".all")) { - const { start_seq = 0 } = this.env.jc.decode(raw.data) ?? {}; - for (const [m] of this.raw) { - if (m.seq >= start_seq) { - raw.respond(m.data, { headers: m.headers }); - } - } - raw.respond(Empty); - continue; - } else if (raw.subject.endsWith(".send")) { - let options: any = undefined; - if (raw.headers) { - for (const [key, value] of raw.headers) { - if (key == COCALC_OPTIONS_HEADER) { - options = JSON.parse(value[0]); - break; - } - } - } - let resp; - try { - resp = await this.sendAsLeader(raw.data, options); - } catch (err) { - raw.respond(this.env.jc.encode({ error: `${err}` })); - return; - } - raw.respond(this.env.jc.encode(resp)); - continue; - } - } - }; - - private sendHeartbeats = async () => { - while (this.env != null) { - await waitUntilConnected(); - const now = Date.now(); - const wait = this.heartbeatInterval - (now - this.lastHeartbeat); - if (wait > 100) { - await delay(wait); - } else { - const now = Date.now(); - this.env.nc.publish(this.subject, Empty); - this.lastHeartbeat = now; - await delay(this.heartbeatInterval); - } - } - }; - - private heartbeatMonitor = async () => { - while (this.env != null) { - if (Date.now() - this.lastHeartbeat >= 2.1 * this.heartbeatInterval) { - try { - // console.log("skipped a heartbeat -- reconnecting"); - await this.reconnect(); - } catch {} - } - await delay(this.heartbeatInterval); - } - }; - - private listen = async () => { - await waitUntilConnected(); - if (this.env == null) { - return; - } - while (this.env != null) { - // @ts-ignore - this.sub?.close(); - this.sub = this.env.nc.subscribe(this.subject); - try { - for await (const raw0 of this.sub) { - if (!this.leader) { - this.lastHeartbeat = Date.now(); - } - if (raw0.data.length == 0 && raw0.headers == null) { - // console.log("received heartbeat"); - // it's a heartbeat probe - continue; - } - const raw = getRawMsg(raw0); - if ( - !this.leader && - this.sessionId && - this.sessionId != raw.sessionId - ) { - await this.reset(); - return; - } else if ( - !this.leader && - this.lastSeq && - raw.seq > this.lastSeq + 1 - ) { - // console.log("skipped a sequence number - reconnecting"); - await this.reconnect(); - return; - } else if (raw.seq <= this.lastSeq) { - // already saw this - continue; - } - if (!this.sessionId) { - this.sessionId = raw.sessionId; - } - // move sequence number forward one and record the data - this.lastSeq = raw.seq; - const mesg = this.decodeValue(raw.data); - this.messages.push(mesg); - this.raw.push([raw]); - this.lastSeq = raw.seq; - this.emit("change", mesg, [raw]); - } - } catch (err) { - console.log(`Error listening -- ${err}`); - } - await delay(3000); - } - this.enforceLimits(); - }; - - private reconnect = reuseInFlight(async () => { - if (this.leader) { - // leader doesn't have a notion of reconnect -- it is the one that - // gets connected to - return; - } - // @ts-ignore - this.sub?.close(); - delete this.sub; - await this.getAllFromLeader({ start_seq: this.lastSeq + 1, noEmit: false }); - this.listen(); - }); - - private encodeValue = (value: T) => { - if (this.env == null) { - throw Error("closed"); - } - return this.valueType == "json" ? this.env.jc.encode(value) : value; - }; - - private decodeValue = (value): T => { - if (this.env == null) { - throw Error("closed"); - } - return this.valueType == "json" ? this.env.jc.decode(value) : value; - }; - - publish = async ( - mesg: T, - options?: { headers?: { [key: string]: string }; msgID?: string }, - ) => { - const data = this.encodeValue(mesg); - - // this may throw an exception: - enforceRateLimits({ - limits: this.limits, - bytesSent: this.bytesSent, - subject: this.subject, - data, - mesg, - }); - - if (this.leader) { - // sending from leader -- so assign seq, timestamp and sent it out. - return await this.sendAsLeader(data, options); - } else { - const timeout = 15000; // todo - // sending as non-leader -- ask leader to send it. - let headers; - if (options != null && Object.keys(options).length > 0) { - headers = createHeaders(); - headers.append(COCALC_OPTIONS_HEADER, JSON.stringify(options)); - } else { - headers = undefined; - } - await waitUntilConnected(); - if (this.env == null) { - throw Error("closed"); - } - const resp = await this.env.nc.request(this.subject + ".send", data, { - headers, - timeout, - }); - const r = this.env.jc.decode(resp.data); - if (r.error) { - throw Error(r.error); - } - return resp; - } - }; - - private sendAsLeader = async (data, options?): Promise<{ seq: number }> => { - if (!this.leader) { - throw Error("must be the leader"); - } - const seq = this.seq; - this.seq += 1; - const f = (cb) => { - this.sendQueue.push({ data, options, seq, cb }); - this.processQueue(); - }; - await callback(f); - return { seq }; - }; - - private processQueue = reuseInFlight(async () => { - if (!this.leader) { - throw Error("must be the leader"); - } - const { sessionId } = this; - while ( - this.sendQueue.length > 0 && - this.env != null && - this.sessionId == sessionId - ) { - const x = this.sendQueue.shift(); - if (x == null) { - continue; - } - const { data, options, seq, cb } = x; - if (options?.msgID && this.msgIDs.has(options?.msgID)) { - // it's a dup of one already successfully sent before -- dedup by ignoring. - cb(); - continue; - } - await waitUntilConnected(); - if (this.env == null) { - cb("closed"); - return; - } - const timestamp = Date.now(); - const headers = createHeaders(); - if (options?.headers) { - for (const k in options.headers) { - headers.append(k, `${options.headers[k]}`); - } - } - headers.append(COCALC_SEQUENCE_HEADER, `${seq}`); - headers.append(COCALC_TIMESTAMP_HEADER, `${timestamp}`); - if (!this.sessionId) { - throw Error("sessionId must be set"); - } - headers.append(COCALC_SESSION_ID_HEADER, this.sessionId); - if (options?.msgID) { - headers.append(COCALC_MESSAGE_ID_HEADER, options.msgID); - } - // we publish it until we get it as a change event, and only - // then do we respond, being sure it was sent. - const now = Date.now(); - while (this.env != null && this.sessionId == sessionId) { - this.env.nc.publish(this.subject, data, { headers }); - const start = Date.now(); - let done = false; - try { - while ( - Date.now() - start <= PUBLISH_TIMEOUT && - this.sessionId == sessionId - ) { - const [_, raw] = await once(this, "change", PUBLISH_TIMEOUT); - if (last(raw)?.seq == seq) { - done = true; - break; - } - } - if (done && options?.msgID) { - this.msgIDs.add(options.msgID); - } - cb(done ? undefined : "timeout"); - break; - } catch (err) { - console.warn(`Error processing sendQueue -- ${err}`); - cb(`${err}`); - break; - } - } - if (now > this.lastHeartbeat) { - this.lastHeartbeat = now; - } - } - }); - - get = (n?): T | T[] => { - if (n == null) { - return this.getAll(); - } else { - return this.messages[n]; - } - }; - - getAll = () => { - return [...this.messages]; - }; - - get length(): number { - return this.messages.length; - } - - get start_seq(): number | undefined { - return this._start_seq; - } - - headers = (n: number): { [key: string]: string } | undefined => { - return headersFromRawMessages(this.raw[n]); - }; - - // load older messages starting at start_seq - load = async ({ - start_seq, - noEmit, - }: { - start_seq: number; - noEmit?: boolean; - }) => { - if (this._start_seq == null || this._start_seq <= 1 || this.leader) { - // we already loaded everything on initialization; there can't be anything older; - // or we are leader, so we are the full source of truth. - return; - } - // this is NOT efficient - it just discards everything and starts over. - const n = this.messages.length; - this.resetState(); - this._start_seq = start_seq; - this.lastSeq = start_seq - 1; - await this.reconnect(); - if (!noEmit) { - for (let i = 0; i < this.raw.length - n; i++) { - this.emit("change", this.messages[i], this.raw[i]); - } - } - }; - - // get server assigned time of n-th message in stream - time = (n: number): Date | undefined => { - const r = last(this.raw[n]); - if (r == null) { - return; - } - return new Date(r.timestamp); - }; - - times = () => { - const v: (Date | undefined)[] = []; - for (let i = 0; i < this.length; i++) { - v.push(this.time(i)); - } - return v; - }; - - stats = ({ - start_seq = 1, - }: { - start_seq?: number; - }): { count: number; bytes: number } | undefined => { - if (this.raw == null) { - return; - } - let count = 0; - let bytes = 0; - for (const raw of this.raw) { - const seq = last(raw)?.seq; - if (seq == null) { - continue; - } - if (seq < start_seq) { - continue; - } - count += 1; - for (const r of raw) { - bytes += r.data.length; - } - } - return { count, bytes }; - }; - - // delete all messages up to and including the - // one at position index, i.e., this.messages[index] - // is deleted. - // NOTE: other clients will NOT see the result of a purge. - purge = async ({ index = -1 }: { index?: number } = {}) => { - if (index >= this.raw.length - 1 || index == -1) { - index = this.raw.length - 1; - } - this.messages.splice(0, index + 1); - this.raw.splice(0, index + 1); - }; - - private enforceLimitsNow = reuseInFlight(async () => { - const index = enforceLimits({ - messages: this.messages, - raw: this.raw, - limits: this.limits, - }); - if (index > -1) { - try { - // console.log("imposing limit via purge ", { index }); - await this.purge({ index }); - } catch (err) { - if (err.code != "TIMEOUT") { - console.log(`WARNING: purging old messages - ${err}`); - } - } - } - }); - - private enforceLimits = throttle( - this.enforceLimitsNow, - ENFORCE_LIMITS_THROTTLE_MS, - { leading: false, trailing: true }, - ); -} - -export const cache = refCache({ - name: "ephemeral-stream", - createObject: async (options: EphemeralStreamOptions) => { - const estream = new EphemeralStream(options); - await estream.init(); - return estream; - }, -}); -export async function estream( - options: EphemeralStreamOptions, -): Promise> { - return await cache(options); -} - -function getRawMsg(raw: Msg): RawMsg { - let seq = 0, - timestamp = 0, - sessionId = ""; - for (const [key, value] of raw.headers ?? []) { - if (key == COCALC_SEQUENCE_HEADER) { - seq = parseInt(value[0]); - } else if (key == COCALC_TIMESTAMP_HEADER) { - timestamp = parseFloat(value[0]); - } else if (key == COCALC_SESSION_ID_HEADER) { - sessionId = value[0]; - } - } - if (!seq) { - throw Error("missing seq header"); - } - if (!timestamp) { - throw Error("missing timestamp header"); - } - // @ts-ignore - raw.seq = seq; - // @ts-ignore - raw.timestamp = timestamp; - // @ts-ignore - raw.sessionId = sessionId; - // @ts-ignore - return raw; -} diff --git a/src/packages/nats/sync/general-dkv.ts b/src/packages/nats/sync/general-dkv.ts deleted file mode 100644 index b4b94d5c2d..0000000000 --- a/src/packages/nats/sync/general-dkv.ts +++ /dev/null @@ -1,507 +0,0 @@ -/* -Eventually Consistent Distributed Key:Value Store - -- You give one or more subjects and this provides a synchronous eventually consistent - "multimaster" distributed way to work with the KV store of keys matching any of those subjects, - inside of the named KV store. -- You should define a 3-way merge function, which is used to automatically resolve all - conflicting writes. The default is to use our local version, i.e., "last write to remote wins". -- All set/get/delete operations are synchronous. -- The state gets sync'd in the backend to NATS as soon as possible. - -This class is based on top of the Consistent Centralized Key:Value Store defined in kv.ts. -You can use the same key:value store at the same time via both interfaces, and if store -is a DKV, you can also access the underlying KV via "store.kv". - -- You must explicitly call "await store.init()" to initialize this before using it. - -- The store emits an event ('change', key) whenever anything changes. - -- Calling "store.getAll()" provides ALL the data, and "store.get(key)" gets one value. - -- Use "store.set(key,value)" or "store.set({key:value, key2:value2, ...})" to set data, - with the following semantics: - - - in the background, changes propagate to NATS. You do not do anything explicitly and - this should never raise an exception. - - - you can call "store.hasUnsavedChanges()" to see if there are any unsaved changes. - - - call "store.unsavedChanges()" to see the unsaved keys. - -- The 3-way merge function takes as input {local,remote,prev,key}, where - - key = the key where there's a conflict - - local = your version of the value - - remote = the remote value, which conflicts in that isEqual(local,remote) is false. - - prev = a known common prev of local and remote. - - (any of local, remote or prev can be undefined, e.g., no previous value or a key was deleted) - - You can do anything synchronously you want to resolve such conflicts, i.e., there are no - axioms that have to be satisifed. If the 3-way merge function throws an exception (or is - not specified) we silently fall back to "last write wins". - - -DEVELOPMENT: - -~/cocalc/src/packages/server$ node -Welcome to Node.js v18.17.1. -Type ".help" for more information. -> env = await require("@cocalc/backend/nats/env").getEnv(); a = require("@cocalc/nats/sync/dkv"); s = new a.DKV({name:'test',env,filter:['foo.>'],merge:({local,remote})=>{return {...remote,...local}}}); await s.init(); - - -In the browser console: - -> s = await cc.client.nats_client.dkv({filter:['foo.>'],merge:({local,remote})=>{return {...remote,...local}}}) - -# NOTE that the name is account-{account_id} or project-{project_id}, -# and if not given defaults to the account-{user's account id} -> s.kv.name -'account-6aae57c6-08f1-4bb5-848b-3ceb53e61ede' - -> s.on('change',(key)=>console.log(key));0; - - -TODO: - - require not-everything subject or have an explicit size limit? - - some history would be VERY useful here due to the merge conflicts. - - for conflict resolution maybe instead of local and remote, just give - two values along with their assigned sequence numbers (?). I.e., something - where the resolution doesn't depend on where it is run. ? Or maybe this doesn't matter. -*/ - -import { EventEmitter } from "events"; -import { GeneralKV, type KVLimits } from "./general-kv"; -import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; -import { type NatsEnv, type ValueType } from "@cocalc/nats/types"; -import { isEqual } from "lodash"; -import { delay } from "awaiting"; -import { map as awaitMap } from "awaiting"; -import { getClient, type ClientWithState } from "@cocalc/nats/client"; - -export const TOMBSTONE = Symbol("tombstone"); -const MAX_PARALLEL = 250; - -export type MergeFunction = (opts: { - key: string; - prev: any; - local: any; - remote: any; -}) => any; - -interface Options { - headers?: { [name: string]: string | null }; -} - -export class GeneralDKV extends EventEmitter { - private kv?: GeneralKV; - private jc?; - private merge?: MergeFunction; - private local: { [key: string]: T | typeof TOMBSTONE } = {}; - private options: { [key: string]: Options } = {}; - private saved: { [key: string]: T | typeof TOMBSTONE } = {}; - private changed: Set = new Set(); - private noAutosave: boolean; - private client?: ClientWithState; - public readonly valueType: ValueType; - public readonly name: string; - public readonly desc?: string; - - constructor({ - name, - env, - filter, - merge, - options, - noAutosave, - limits, - valueType, - desc, - }: { - name: string; - // used for log and error messages - desc: string; - env: NatsEnv; - // 3-way merge conflict resolution - merge?: (opts: { - key: string; - prev?: any; - local?: any; - remote?: any; - }) => any; - // filter: optionally restrict to subset of named kv store matching these subjects. - // NOTE: any key name that you *set or delete* must match one of these - filter: string | string[]; - limits?: KVLimits; - // if noAutosave is set, local changes are never saved until you explicitly - // call "await this.save()", which will try once to save. Changes made during - // the save may not be saved though. - noAutosave?: boolean; - options?; - valueType?: ValueType; - }) { - super(); - this.name = name; - this.desc = desc; - this.merge = merge; - this.noAutosave = !!noAutosave; - this.jc = env.jc; - this.valueType = valueType ?? "json"; - this.kv = new GeneralKV({ name, env, filter, options, limits, valueType }); - if (!noAutosave) { - this.client = getClient(); - this.client.on("connected", this.save); - } - } - - init = reuseInFlight(async () => { - if (this.kv == null) { - throw Error("closed"); - } - this.kv.on("change", this.handleRemoteChange); - await this.kv.init(); - this.emit("connected"); - }); - - close = async () => { - if (this.kv == null) { - return; - } - if (!this.noAutosave) { - try { - await this.save(); - } catch (err) { - // [ ] TODO: try localStorage or a file?! throw? - console.log( - `WARNING: unable to save some data when closing a general-dkv -- ${err}`, - ); - } - this.client?.removeListener("connected", this.save); - } - this.kv.close(); - this.emit("closed"); - this.removeAllListeners(); - delete this.kv; - // @ts-ignore - delete this.local; - // @ts-ignore - delete this.options; - // @ts-ignore - delete this.changed; - delete this.merge; - }; - - private discardLocalState = (key: string) => { - delete this.local[key]; - delete this.options[key]; - delete this.saved[key]; - if (this.isStable()) { - this.emit("stable"); - } - }; - - // stable = everything is saved *and* also echoed back from the server as confirmation. - isStable = () => { - for (const _ in this.local) { - return false; - } - return true; - }; - - private handleRemoteChange = ({ key, value: remote, prev }) => { - const local = this.local[key] === TOMBSTONE ? undefined : this.local[key]; - let value: any = remote; - if (local !== undefined) { - // we have an unsaved local value, so let's check to see if there is a - // conflict or not. - if (isEqual(local, remote)) { - // incoming remote value is equal to unsaved local value, so we can - // just discard our local value (no need to save it). - this.discardLocalState(key); - } else { - // There is a conflict. Let's resolve the conflict: - // console.log("merge conflict", { key, remote, local, prev }); - try { - value = this.merge?.({ key, local, remote, prev }) ?? local; - // console.log("merge conflict --> ", value); - // console.log("handle merge conflict", { - // key, - // local, - // remote, - // prev, - // value, - // }); - } catch (err) { - console.warn("exception in merge conflict resolution", err); - // user provided a merge function that throws an exception. We select local, since - // it is the newest, i.e., "last write wins" - value = local; - // console.log("merge conflict ERROR --> ", err, value); - } - if (isEqual(value, remote)) { - // no change, so forget our local value - this.discardLocalState(key); - } else { - // resolve with the new value, or if it is undefined, a TOMBSTONE, - // meaning choice is to delete. - // console.log("conflict resolution: ", { key, value }); - if (value === TOMBSTONE) { - this.delete(key); - } else { - this.set(key, value); - } - } - } - } - this.emit("change", { key, value, prev }); - }; - - get = (key: string): T | undefined => { - if (this.kv == null) { - throw Error("closed"); - } - this.assertValidKey(key); - const local = this.local[key]; - if (local === TOMBSTONE) { - return undefined; - } - if (local !== undefined) { - return local; - } - return this.kv.get(key); - }; - - get length(): number { - // not efficient - return Object.keys(this.getAll()).length; - } - - getAll = (): { [key: string]: T } => { - if (this.kv == null) { - throw Error("closed"); - } - const x = { ...this.kv.getAll(), ...this.local }; - for (const key in this.local) { - if (this.local[key] === TOMBSTONE) { - delete x[key]; - } - } - return x as { [key: string]: T }; - }; - - has = (key: string): boolean => { - if (this.kv == null) { - throw Error("closed"); - } - const a = this.local[key]; - if (a === TOMBSTONE) { - return false; - } - if (a !== undefined) { - return true; - } - return this.kv.has(key); - }; - - time = (key?: string): { [key: string]: Date } | Date | undefined => { - if (this.kv == null) { - throw Error("closed"); - } - return this.kv.time(key); - }; - - private assertValidKey = (key): void => { - if (this.kv == null) { - throw Error("closed"); - } - this.kv.assertValidKey(key); - }; - - private _delete = (key) => { - this.local[key] = TOMBSTONE; - this.changed.add(key); - }; - - delete = (key) => { - this.assertValidKey(key); - this._delete(key); - if (!this.noAutosave) { - this.save(); - } - }; - - clear = () => { - if (this.kv == null) { - throw Error("closed"); - } - for (const key in this.kv.getAll()) { - this._delete(key); - } - for (const key in this.local) { - this._delete(key); - } - if (!this.noAutosave) { - this.save(); - } - }; - - private toValue = (obj) => { - if (obj === undefined) { - return TOMBSTONE; - } - if (this.valueType == "binary") { - if (!ArrayBuffer.isView(obj)) { - throw Error("value must be an array buffer"); - } - return obj; - } - // It's EXTREMELY important that anything we save to NATS has the property that - // jc.decode(jc.encode(obj)) is the identity map. That is very much NOT - // the case for stuff that set gets called on, e.g., {a:new Date()}. - // Thus before storing it in in any way, we ensure this immediately: - return this.jc.decode(this.jc.encode(obj)); - }; - - headers = (key: string): { [key: string]: string } | undefined => { - return this.kv?.headers(key); - }; - - set = (key: string, value: T, options?: Options) => { - this.assertValidKey(key); - const obj = this.toValue(value); - this.local[key] = obj; - if (options != null) { - this.options[key] = options; - } - this.changed.add(key); - if (!this.noAutosave) { - this.save(); - } - }; - - setMany = (obj) => { - for (const key in obj) { - this.assertValidKey(key); - this.local[key] = this.toValue(obj[key]); - this.changed.add(key); - } - if (!this.noAutosave) { - this.save(); - } - }; - - hasUnsavedChanges = () => { - if (this.kv == null) { - return false; - } - return this.unsavedChanges().length > 0; - }; - - unsavedChanges = () => { - return Object.keys(this.local).filter( - (key) => this.local[key] !== this.saved[key], - ); - }; - - save = reuseInFlight(async () => { - if (this.noAutosave) { - return await this.attemptToSave(); - // one example error when there's a conflict brewing: - /* - { - code: 10071, - name: 'JetStreamApiError', - message: 'wrong last sequence: 84492' - } - */ - } - let d = 100; - while (true) { - let status; - try { - status = await this.attemptToSave(); - //console.log("successfully saved"); - } catch (_err) { - //console.log("temporary issue saving", this.kv?.name, _err); - } - if (!this.hasUnsavedChanges()) { - return status; - } - d = Math.min(10000, d * 1.3) + Math.random() * 100; - await delay(d); - } - }); - - private attemptToSave = reuseInFlight(async () => { - if (this.kv == null) { - throw Error("closed"); - } - this.changed.clear(); - const status = { unsaved: 0, set: 0, delete: 0 }; - const obj = { ...this.local }; - for (const key in obj) { - if (obj[key] === TOMBSTONE) { - status.unsaved += 1; - await this.kv.delete(key); - status.delete += 1; - status.unsaved -= 1; - delete obj[key]; - if (!this.changed.has(key)) { - // successfully saved this and user didn't make a change *during* the set - this.discardLocalState(key); - } - } - } - const f = async (key) => { - if (this.kv == null) { - // closed - return; - } - try { - status.unsaved += 1; - await this.kv.set(key, obj[key] as T, this.options[key]); - // console.log("kv store -- attemptToSave succeed", this.desc, { - // key, - // value: obj[key], - // }); - status.unsaved -= 1; - status.set += 1; - if (!this.changed.has(key)) { - // successfully saved this and user didn't make a change *during* the set - this.discardLocalState(key); - } - // note that we CANNOT call this.discardLocalState(key) here, because - // this.get(key) needs to work immediately after save, but if this.local[key] - // is deleted, then this.get(key) would be undefined, because - // this.kv.get(key) only has value in it once the value is - // echoed back from the server. - } catch (err) { - // console.log("kv store -- attemptToSave failed", this.desc, err, { - // key, - // value: obj[key], - // }); - if (err.code == "REJECT" && err.key) { - const value = this.local[err.key]; - // can never save this. - this.discardLocalState(err.key); - status.unsaved -= 1; - this.emit("reject", { key: err.key, value }); - } - if ( - err.code == "10071" && - err.message.startsWith("wrong last sequence") - ) { - // this happens when another client has published a NEWER version of this key, - // so the right thing is to just ignore this. In a moment there will be no - // need to save anything, since we'll receive a message that overwrites this key. - return; - } - throw err; - } - }; - await awaitMap(Object.keys(obj), MAX_PARALLEL, f); - return status; - }); - - stats = () => this.kv?.stats(); -} diff --git a/src/packages/nats/sync/general-kv.ts b/src/packages/nats/sync/general-kv.ts deleted file mode 100644 index 2dfd9c07c4..0000000000 --- a/src/packages/nats/sync/general-kv.ts +++ /dev/null @@ -1,1199 +0,0 @@ -/* -Async Consistent Centralized Key Value Store - -- You give one or more subjects and this provides an asynchronous but consistent - way to work with the KV store of keys matching any of those subjects, - inside of the named KV store. -- The get operation is sync. (It can of course be slightly out of date, but that is detected - if you try to immediately write it.) -- The set will fail if the local cached value (returned by get) turns out to be out of date. -- Also delete and set will fail if the NATS connection is down or times out. -- For an eventually consistent sync wrapper around this, use DKV, defined in the sibling file dkv.ts. - -WARNING: Nats itself actually currently seems to have no model for consistency, especially -with multiple nodes. See https://github.com/nats-io/nats-server/issues/6557 - -This is a simple KV wrapper around NATS's KV, for small KV stores. Each client holds a local cache -of all data, which is used to ensure set's are a no-op if there is no change. Also, this automates -ensuring that if you do a read-modify-write, this will succeed only if nobody else makes a change -before you. - -- You must explicitly call "await store.init()" to initialize it before using it. - -- The store emits an event ('change', key, newValue, previousValue) whenever anything changes - -- Calling "store.get()" provides ALL the data and is synchronous. It uses various API tricks to - ensure this is fast and is updated when there is any change from upstream. Use "store.get(key)" - to get the value of one key. - -- Use "await store.set(key,value)" or "await store.set({key:value, key2:value2, ...})" to set data, - with the following semantics: - - - set ONLY makes a change if our local version ("store.get(key)") of the value is different from - what you're trying to set the value to, where different is defined by lodash isEqual. - - - if our local version this.get(key) was not the most recent version in NATS, then the set will - definitely throw an exception! This is fantastic because it means you can modify and save what - is in the local cache on multiple nodes at once anywhere, and be 100% certain to never overwrite - data in complicated objects. Of course, you have to assume "await store.set(...)" WILL - sometimes fail. - - - Set with multiple keys "pipelines" in that MAX_PARALLEL key/value pairs are set at once, without - waiting for every single individual set to get ACK'd from the server before doing more sets. - This makes this **massively** faster, but means that if "await store.set(...)" fails, you don't - immediately know which keys were successfully set and which failed, though all keys worked will get - updated soon and reflected in store.get(). - -- Use "await store.expire(ageMs)" to delete every key that was last changed at least ageMs - milliseconds in the past. - - TODO/WARNING: the timestamps are defined by NATS (and its clock), but - the definition of "ageMs in the past" is defined by the client where this is called. Thus - if the client's clock is off, that would be a huge problem. An obvious solution is to - get the current time from NATS, and use that. I don't know a "good" way to get the current - time except maybe publishing a message to myself...? - - -CHUNKING: - - -Similar to streams, unlike NATS itself, hwere we allow storing arbitrarily large -values, in particular, values that could be much larger than the configured message -size. When doing a set if the value exceeds the limit, we store the part of -the value that fits, and store a *header* that describes where the rest of the -values are stored. For a given key, the extra chunks are stored with keys: - - ${key}.${i}.chunk - -When receiving changes, these extra chunks are temporarily kept separately, -then used to compute the value for key. All other paramaters, e.g., sequence -numbers, last time, etc., use the main key. - -TODO: - -- [ ] maybe expose some functionality related to versions/history? - -DEVELOPMENT: - -(See packages/backend/nats/test/sync/general-kv.test.ts for a unit tested version of what is below that -actually works.) - -~/cocalc/src/packages/server$ n -Welcome to Node.js v18.17.1. -Type ".help" for more information. -> env = await require("@cocalc/backend/nats/env").getEnv(); a = require("@cocalc/nats/sync/general-kv"); s = new a.GeneralKV({name:'test',env,filter:['foo.>']}); await s.init(); - -> await s.set("foo.x", 10) -> s.getAll() -{ 'foo.x': 10 } -> await s.delete("foo.x") -undefined -> s.getAll() -{} -> await s.set("foo.x", 10) - -// Since the filters are disjoint these are totally different: - -> t = new a.GeneralKV({name:'test2',env,filter:['bar.>']}); await t.init(); -> await t.getAll() -{} -> await t.set("bar.abc", 10) -undefined -> await t.getAll() -{ 'bar.abc': 10} -> await s.getAll() -{ 'foo.x': 10 } - -// The union: -> u = new a.GeneralKV({name:'test3',env,filter:['bar.>', 'foo.>']}); await u.init(); -> u.getAll() -{ 'foo.x': 10, 'bar.abc': 10 } -> await s.set('foo.x', 999) -undefined -> u.getAll() -{ 'bar.abc': 10, 'foo.x': 999} -*/ - -import { EventEmitter } from "events"; -import { type NatsEnv } from "@cocalc/nats/types"; -import { Kvm } from "@nats-io/kv"; -import { getAllFromKv, matchesPattern, getMaxPayload } from "@cocalc/nats/util"; -import { isEqual } from "lodash"; -import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; -import { map as awaitMap } from "awaiting"; -import { throttle } from "lodash"; -import { delay } from "awaiting"; -import { headers as createHeaders } from "@nats-io/nats-core"; -import type { MsgHdrs } from "@nats-io/nats-core"; -import type { ValueType } from "@cocalc/nats/types"; -import { isConnected, waitUntilConnected } from "@cocalc/nats/util"; -import { ENFORCE_LIMITS_THROTTLE_MS } from "./stream"; -import { asyncDebounce } from "@cocalc/util/async-utils"; -import { waitUntilReady } from "@cocalc/nats/tiered-storage/client"; - -const PUBLISH_TIMEOUT = 15000; - -class RejectError extends Error { - code: string; - key: string; -} - -const MAX_PARALLEL = 250; - -const CONNECTION_CHECK_INTERVAL = 5000; - -// Note that the limit options are named in exactly the same was as for streams, -// which is convenient for consistency. This is not consistent with NATS's -// own KV store limit naming. - -export interface KVLimits { - // How many keys may be in the KV store. Oldest keys will be removed - // if the key-value store exceeds this size. -1 for unlimited. - max_msgs: number; - - // Maximum age of any key, expressed in milliseconds. 0 for unlimited. - // Age is updated whenever value of the key is changed. - max_age: number; - - // The maximum number of bytes to store in this KV, which means - // the total of the bytes used to store everything. Since we store - // the key with each value (to have arbitrary keys), this includes - // the size of the keys. - max_bytes: number; - - // The maximum size of any single value, including the key. - max_msg_size: number; -} - -export class GeneralKV extends EventEmitter { - public readonly name: string; - private options?; - private filter?: string[]; - private env: NatsEnv; - private kv?; - private watch?; - private all?: { [key: string]: T }; - private revisions?: { [key: string]: number }; - private chunkCounts: { [key: string]: number } = {}; - private times?: { [key: string]: Date }; - private sizes?: { [key: string]: number }; - private allHeaders: { [key: string]: MsgHdrs } = {}; - private limits: KVLimits; - private revision: number = 0; - public readonly valueType: ValueType; - private noWatch: boolean; - private noGet: boolean; - - constructor({ - name, - env, - filter, - options, - limits, - valueType, - noWatch, - noGet, - }: { - name: string; - // filter: optionally restrict to subset of named kv store matching these subjects. - // NOTE: any key name that you *set or delete* should match one of these - filter?: string | string[]; - env: NatsEnv; - options?; - limits?: Partial; - valueType?: ValueType; - noWatch?: boolean; - noGet?: boolean; - }) { - super(); - this.limits = { - max_msgs: -1, - max_age: 0, - max_bytes: -1, - max_msg_size: -1, - ...limits, - }; - - this.noWatch = !!noWatch; - this.noGet = !!noGet; - this.env = env; - this.name = name; - this.options = options; - this.filter = typeof filter == "string" ? [filter] : filter; - this.valueType = valueType ?? "json"; - if (this.valueType != "json" && this.valueType != "binary") { - throw Error("valueType must be 'json' or 'binary'"); - } - } - - init = reuseInFlight(async () => { - if (this.all != null) { - return; - } - await waitUntilReady(this.name); - const kvm = new Kvm(this.env.nc); - await waitUntilConnected(); - this.kv = await kvm.create(this.name, { - compression: true, - ...this.options, - }); - this.kv.validateKey = validateKey; - this.kv.validateSearchKey = validateSearchKey; - if (this.noGet) { - this.times = {}; - this.revisions = {}; - this.allHeaders = {}; - this.chunkCounts = {}; - this.sizes = {}; - this.all = {}; - this.revision = 0; - return; - } - - await waitUntilConnected(); - const { all, revisions, times, headers } = await getAllFromKv({ - kv: this.kv, - key: this.filter, - }); - this.revisions = revisions; - this.times = times; - this.allHeaders = {}; - this.chunkCounts = {}; - this.sizes = {}; - const usedKeys = new Set(); - const all0: { [key: string]: T } = {}; - const chunkData: { - [key: string]: { chunkCount?: number; chunks: Buffer[] }; - } = {}; - for (const key in all) { - let value: Buffer | null = null; - const chunkCount = getChunkCount(headers[key]); - let chunkKey: string = ""; - let key0 = ""; - if (chunkCount) { - if (chunkData[key] == null) { - chunkData[key] = { chunkCount, chunks: [all[key]] }; - } else { - chunkData[key].chunkCount = chunkCount; - chunkData[key].chunks[0] = all[key]; - } - chunkKey = key; - this.allHeaders[key] = headers[key]; - } else if (isChunkedKey(key)) { - delete this.times[key]; - delete this.revisions[key]; - const { key: ckey, index } = parseChunkedKey(key); - chunkKey = ckey; - if (chunkData[chunkKey] == null) { - chunkData[chunkKey] = { chunks: [] }; - } - chunkData[chunkKey].chunks[index] = all[key]; - } else { - key0 = key; - value = all[key]; - usedKeys.add(key0); - this.allHeaders[key] = headers[key]; - } - - if (chunkKey && chunkData[chunkKey].chunkCount != null) { - let i = 0; - for (const chunk of chunkData[chunkKey].chunks) { - if (chunk !== undefined) { - i += 1; - } - } - const { chunkCount } = chunkData[chunkKey]; - if (i >= chunkCount!) { - // nextjs prod complains about this... - // @ts-ignore - value = Buffer.concat(chunkData[chunkKey].chunks); - key0 = chunkKey; - this.chunkCounts[key0] = chunkCount!; - delete chunkData[chunkKey]; - usedKeys.add(chunkKey); - for (let chunk = 1; chunk < chunkCount!; chunk++) { - usedKeys.add(chunkedKey({ key: chunkKey, chunk })); - } - } - } - - if (value == null) { - continue; - } - this.sizes[key0] = value.length; - try { - all0[key0] = this.decode(value); - } catch (err) { - // invalid json -- corruption. I hit this ONLY when doing development - // and explicitly putting bad data in. This isn't normal. But it's - // help to make this a warning, in order to not make all data not accessible. - console.warn(`WARNING: unable to read ${key0} -- ${err}`); - } - } - this.all = all0; - this.revision = Math.max(0, ...Object.values(this.revisions)); - this.emit("connected"); - if (!this.noWatch) { - this.startWatchLoop(); - this.monitorWatch(); - } - - // Also anything left at this point is garbage that needs to be freed: - for (const key in all) { - if (!usedKeys.has(key)) { - await this.kv.delete(key); - } - } - }); - - private encode = (value) => { - return this.valueType == "json" ? this.env.jc.encode(value) : value; - }; - - private decode = (value) => { - return this.valueType == "json" ? this.env.jc.decode(value) : value; - }; - - private restartWatch = () => { - // this triggers the end of the "for await (const x of this.watch) {" - // loop in startWatch, which results in another watch starting, - // assuming the object isn't closed. - this.watch?.stop(); - }; - - private startWatchLoop = async () => { - let d = 1000; - let lastTime = 0; - while (this.all != null) { - if (Date.now() - lastTime > 60 * 1000) { - // reset delay if it has been a while -- delay is only to prevent frequent bursts - d = 1000; - } - try { - await waitUntilConnected(); - if (this.all == null) { - return; - } - const resumeFromRevision = this.revision - ? this.revision + 1 - : undefined; - await this.startWatch({ resumeFromRevision }); - d = 1000; - } catch (_err) { - // expected to happen sometimes, e.g., when the connection closes temporarily - // if (err.code != "CONNECTION_CLOSED") { - // console.log(`WARNING: getting watch on kv...`); - // } - } - if (this.all == null) { - // closed - return; - } - d = Math.min(20000, d * 1.25) + Math.random(); - // console.log(`waiting ${d}ms then reconnecting`); - await delay(d); - } - }; - - private startWatch = async ({ - resumeFromRevision, - }: { resumeFromRevision?: number } = {}) => { - // watch for changes, starting AFTER the last revision we retrieved - this.watch = await this.kv.watch({ - ignoreDeletes: false, - include: "updates", - key: this.filter, - resumeFromRevision, - }); - const chunkData: { - [key: string]: { - chunkCount?: number; - chunks: Buffer[]; - revision?: number; - }; - } = {}; - for await (const x of this.watch) { - const { revision, key, value, sm } = x; - this.revision = revision; - if ( - this.revisions == null || - this.all == null || - this.times == null || - this.sizes == null - ) { - return; - } - - let value0: Buffer | null = null; - const chunkCount = getChunkCount(sm.headers); - let chunkKey: string = ""; - let key0 = ""; - let revision0 = 0; - if (chunkCount) { - if (chunkData[key] == null) { - chunkData[key] = { chunkCount, chunks: [value], revision }; - } else { - chunkData[key].chunkCount = chunkCount; - chunkData[key].chunks[0] = value; - chunkData[key].revision = revision; - } - chunkKey = key; - this.allHeaders[key] = sm.headers; - } else if (isChunkedKey(key)) { - const { key: ckey, index } = parseChunkedKey(key); - chunkKey = ckey; - if (chunkData[chunkKey] == null) { - chunkData[chunkKey] = { chunks: [] }; - } - chunkData[chunkKey].chunks[index] = value; - } else { - key0 = key; - value0 = value; - revision0 = revision; - if (value.length != 0) { - // NOTE: we *only* set the header to remote when not deleting the key. Deleting - // it would delete the header, which contains the actual non-hashed key. - this.allHeaders[key] = sm.headers; - } - delete this.chunkCounts[key0]; - } - - if (chunkKey && chunkData[chunkKey].chunkCount != null) { - let i = 0; - for (const chunk of chunkData[chunkKey].chunks) { - if (chunk !== undefined) { - i += 1; - } - } - const { chunkCount } = chunkData[chunkKey]; - if (i >= chunkCount!) { - // @ts-ignore (for nextjs prod build) - value0 = Buffer.concat(chunkData[chunkKey].chunks); - key0 = chunkKey; - const r = chunkData[chunkKey].revision; - if (r == null) { - throw Error("bug"); - } - revision0 = r; - this.chunkCounts[chunkKey] = chunkCount!; - delete chunkData[chunkKey]; - } - } - - if (value0 == null) { - continue; - } - this.revisions[key0] = revision0; - const prev = this.all[key0]; - if (value0.length == 0) { - // delete - delete this.all[key0]; - delete this.times[key0]; - delete this.sizes[key0]; - delete this.chunkCounts[key0]; - } else { - this.all[key0] = this.decode(value0); - this.times[key0] = sm.time; - this.sizes[key0] = value0.length; - } - this.emit("change", { key: key0, value: this.all[key0], prev }); - this.enforceLimits(); - } - }; - - private monitorWatch = async () => { - if (this.env.nc.on != null) { - this.env.nc.on("reconnect", this.restartWatch); - this.env.nc.on("status", ({ type }) => { - if (type == "reconnect") { - this.ensureWatchIsValid(); - } - }); - } else { - this.checkWatchOnReconnect(); - } - while (this.revisions != null) { - if (!(await isConnected())) { - await waitUntilConnected(); - //console.log("monitorWatch: not connected so restart", this.name); - await this.restartWatch(); - } - //console.log("monitorWatch: wait", this.name); - await delay(CONNECTION_CHECK_INTERVAL); - } - }; - - private ensureWatchIsValid = asyncDebounce( - async () => { - await waitUntilConnected(); - await delay(2000); - const isValid = await this.isWatchStillValid(); - if (!isValid) { - if (this.kv == null) { - return; - } - console.log(`nats kv: ${this.name} -- watch not valid, so recreating`); - await this.restartWatch(); - } - }, - 3000, - { leading: false, trailing: true }, - ); - - private isWatchStillValid = async () => { - await waitUntilConnected(); - if (this.kv == null || this.watch == null) { - return false; - } - try { - await this.watch._data.info(); - return true; - } catch (err) { - console.log(`nats: watch info error -- ${err}`); - return false; - } - }; - - private checkWatchOnReconnect = async () => { - while (this.kv != null) { - try { - for await (const { type } of await this.env.nc.status()) { - if (type == "reconnect") { - await this.ensureWatchIsValid(); - } - } - } catch { - await delay(15000); - await this.ensureWatchIsValid(); - } - } - }; - - close = () => { - if (this.revisions == null) { - // already closed - return; - } - delete this.all; - - this.watch?.stop(); - delete this.watch; - - delete this.times; - delete this.revisions; - delete this.sizes; - delete this.kv; - // @ts-ignore - delete this.allHeaders; - this.emit("closed"); - this.removeAllListeners(); - this.env.nc.removeListener?.("reconnect", this.restartWatch); - }; - - headers = (key: string): { [key: string]: string } | undefined => { - const headers = this.allHeaders?.[key]; - if (headers == null) { - return; - } - const x: { [key: string]: string } = {}; - for (const [key, value] of headers) { - if (key != CHUNKS_HEADER) { - x[key] = value[0]; - } - } - return x; - }; - - // do not use cached this.all - // this is NOT implemented yet if there are chunks! - getDirect = async (key: string): Promise => { - if ( - this.all == null || - this.revisions == null || - this.times == null || - this.sizes == null || - this.allHeaders == null - ) { - throw Error("not initialized"); - } - const x = await this.kv.get(key); - if (x == null) { - return; - } - const { value, revision, sm } = x; - if (value.length == 0) { - return undefined; - } - const v = this.env.jc.decode(value); - this.all[key] = v; - this.revisions[key] = revision; - if (revision > this.revision) { - this.revision = revision; - } - this.times[key] = sm.time; - this.sizes[key] = value.length; - this.allHeaders[key] = sm.headers; - return v; - }; - - get = (key: string): T => { - if (this.all == null) { - throw Error("not initialized"); - } - return this.all[key]; - }; - - getAll = (): { [key: string]: T } => { - if (this.all == null) { - throw Error("not initialized"); - } - return { ...this.all }; - }; - - get length(): number { - if (this.all == null) { - throw Error("not initialized"); - } - return Object.keys(this.all).length; - } - - has = (key: string): boolean => { - return this.all?.[key] !== undefined; - }; - - time = (key?: string): { [key: string]: Date } | Date | undefined => { - if (key == null) { - return this.times; - } else { - return this.times?.[key]; - } - }; - - assertValidKey = (key: string): void => { - if (!this.isValidKey(key)) { - throw Error( - `delete: key (=${key}) must match the filter: ${JSON.stringify(this.filter)}`, - ); - } - }; - - isValidKey = (key: string): boolean => { - if (this.filter == null) { - return true; - } - for (const pattern of this.filter) { - if (matchesPattern({ pattern, subject: key })) { - return true; - } - } - return false; - }; - - seq = (key) => { - if (this.revisions == null) { - throw Error("not ready"); - } - return this.revisions[key]; - }; - - delete = async (key: string, revision?: number) => { - this.assertValidKey(key); - if ( - this.all == null || - this.revisions == null || - this.times == null || - this.sizes == null - ) { - throw Error("not ready"); - } - if (this.all[key] !== undefined || this.noGet || this.noWatch) { - const cur = this.all[key]; - try { - const newRevision = await this.kv.delete(key, { - previousSeq: revision ?? this.revisions[key], - }); - this.revisions[key] = newRevision; - delete this.all[key]; - } catch (err) { - this.all[key] = cur; - throw err; - } - if (this.chunkCounts[key]) { - // garbage collect the extra chunks - for (let chunk = 1; chunk < this.chunkCounts[key]; chunk++) { - await this.kv.delete(chunkedKey({ key, chunk })); - } - delete this.chunkCounts[key]; - } - } - }; - - // delete everything matching the filter that hasn't been set - // in the given amount of ms. Returns number of deleted records. - // NOTE: This could throw an exception if something that would expire - // were changed right when this is run then it would get expired - // but shouldn't. In that case, run it again. - expire = async ({ - cutoff, - ageMs, - }: { - cutoff?: Date; - ageMs?: number; - }): Promise => { - if (!ageMs && !cutoff) { - throw Error("one of ageMs or cutoff must be set"); - } - if (ageMs && cutoff) { - throw Error("exactly one of ageMs or cutoff must be set"); - } - if (this.times == null || this.all == null) { - throw Error("not initialized"); - } - if (ageMs && !cutoff) { - cutoff = new Date(Date.now() - ageMs); - } - if (cutoff == null) { - throw Error("impossible"); - } - // make copy of revisions *before* we start deleting so that - // if a key is changed exactly while deleting we get an error - // and don't accidently delete it! - const revisions = { ...this.revisions }; - const toDelete = Object.keys(this.all).filter( - (key) => this.times?.[key] != null && this.times[key] <= cutoff, - ); - if (toDelete.length > 0) { - await awaitMap(toDelete, MAX_PARALLEL, async (key) => { - await this.delete(key, revisions[key]); - }); - } - return toDelete.length; - }; - - // delete all that we know about - clear = async () => { - if (this.all == null) { - throw Error("not initialized"); - } - await awaitMap(Object.keys(this.all), MAX_PARALLEL, this.delete); - }; - - setMany = async ( - obj: { [key: string]: T }, - headers?: { [key: string]: { [name: string]: string } }, - ) => { - await awaitMap( - Object.keys(obj), - MAX_PARALLEL, - async (key) => await this.set(key, obj[key], headers?.[key]), - ); - }; - - set = async ( - key: string, - value: T, - options?: { - headers?: { [name: string]: string | null }; - previousSeq?: number; - }, - ) => { - await this._set(key, value, options); - if (this.all != null) { - this.all[key] = value; - } - }; - - private _set = async ( - key: string, - value: T, - options?: { - headers?: { [name: string]: string | null }; - previousSeq?: number; - }, - ) => { - if (!this.isValidKey(key)) { - throw Error( - `set: key (=${key}) must match the filter: ${JSON.stringify(this.filter)}`, - ); - } - if (this.all == null || this.revisions == null) { - throw Error("not ready"); - } - if (isEqual(this.all[key], value)) { - // values equal. What about headers? - - if ( - options?.headers == null || - Object.keys(options.headers).length == 0 - ) { - return; - } - const { headers } = options; - // maybe trying to change headers - let changeHeaders = false; - if (this.allHeaders[key] == null) { - // this is null but headers isn't, so definitely trying to change - changeHeaders = true; - } else { - // look to see if any header is explicitly being changed - const keys = new Set(Object.keys(headers)); - for (const [k, v] of this.allHeaders[key]) { - keys.delete(k); - if (headers[k] !== undefined && headers[k] != v[0]) { - changeHeaders = true; - break; - } - } - if (keys.size > 0) { - changeHeaders = true; - } - } - if (!changeHeaders) { - // not changing any header - return; - } - } - if (value === undefined) { - return await this.delete(key); - } - const revision = options?.previousSeq ?? this.revisions[key]; - let val = this.encode(value); - if ( - this.limits.max_msg_size > -1 && - val.length > this.limits.max_msg_size - ) { - // we reject due to our own size reasons - const err = new RejectError( - `message key:value size (=${val.length}) exceeds max_msg_size=${this.limits.max_msg_size} bytes`, - ); - err.code = "REJECT"; - err.key = key; - throw err; - } - - const maxMessageSize = (await getMaxPayload()) - 10000; - // const maxMessageSize = 100; - - if (val.length > maxMessageSize) { - // chunking - let val0 = val; - const chunks: Buffer[] = []; - while (val0.length > 0) { - chunks.push(val0.slice(0, maxMessageSize)); - val0 = val0.slice(maxMessageSize); - } - val = chunks[0]; - let allHeaders = createHeaders(); - allHeaders.append(CHUNKS_HEADER, `${chunks.length}`); - if (options?.headers) { - const { headers } = options; - for (const k in headers) { - const v = headers[k]; - if (v == null) { - continue; - } - allHeaders.append(k, v); - } - } - await jetstreamPut(this.kv, key, val, { - previousSeq: revision, - headers: allHeaders, - timeout: PUBLISH_TIMEOUT, - }); - // now save the other chunks somewhere. - for (let i = 1; i < chunks.length; i++) { - await jetstreamPut(this.kv, chunkedKey({ key, chunk: i }), chunks[i], { - timeout: PUBLISH_TIMEOUT, - }); - } - if (chunks.length < (this.chunkCounts[key] ?? 0)) { - // value previously had even more chunks, so we get rid of the extra chunks. - for ( - let chunk = chunks.length; - chunk < this.chunkCounts[key]; - chunk++ - ) { - await this.kv.delete(chunkedKey({ key, chunk })); - } - } - - this.chunkCounts[key] = chunks.length; - } else { - // not chunking - try { - let allHeaders; - if (options?.headers) { - const { headers } = options; - allHeaders = createHeaders(); - for (const k in headers) { - const v = headers[k]; - if (v == null) { - continue; - } - allHeaders.append(k, v); - } - } else { - allHeaders = undefined; - } - await jetstreamPut(this.kv, key, val, { - previousSeq: revision, - headers: allHeaders, - timeout: PUBLISH_TIMEOUT, - }); - } catch (err) { - if (err.code == "MAX_PAYLOAD_EXCEEDED") { - // nats rejects due to payload size - const err2 = new RejectError(`${err}`); - err2.code = "REJECT"; - err2.key = key; - throw err2; - } else { - throw err; - } - } - if (this.chunkCounts[key]) { - // it was chunked, so get rid of chunks - for (let chunk = 1; chunk < this.chunkCounts[key]; chunk++) { - await this.kv.delete(chunkedKey({ key, chunk })); - } - delete this.chunkCounts[key]; - } - } - }; - - stats = (): { count: number; bytes: number } | undefined => { - if (this.sizes == null) { - return; - } - let count = 0; - let bytes = 0; - for (const key in this.sizes) { - count += 1; - bytes += this.sizes[key]; - } - return { count, bytes }; - }; - - // separated out from throttled version so it's easy to call directly for unit testing. - private enforceLimitsNow = reuseInFlight(async () => { - if (this.all == null || this.times == null || this.sizes == null) { - return; - } - const { max_msgs, max_age, max_bytes } = this.limits; - let times: { time: Date; key: string }[] | null = null; - const getTimes = (): { time: Date; key: string }[] => { - if (times == null) { - // this is potentially a little worrisome regarding performance, but - // it has to be done, or we have to do something elsewhere to maintain - // this info. The intention with these kv's is they are small and all - // in memory. - const v: { time: Date; key: string }[] = []; - for (const key in this.times) { - v.push({ time: this.times[key], key }); - } - v.sort((a, b) => (a.time < b.time ? -1 : a.time > b.time ? 1 : 0)); - times = v; - } - return times!; - }; - - // we check with each defined limit if some old messages - // should be dropped, and if so move limit forward. If - // it is above -1 at the end, we do the drop. - let index = -1; - const setIndex = (i, _limit) => { - // console.log("setIndex", { i, _limit }); - index = Math.max(i, index); - }; - - //max_msgs = max number of keys - const v = Object.keys(this.all); - // console.log("enforceLimitsNow", this.limits, v, getTimes()); - if (max_msgs > -1 && v.length > max_msgs) { - // ensure there are at most this.limits.max_msgs messages - // by deleting the oldest ones up to a specified point. - const i = v.length - max_msgs; - if (i > 0) { - setIndex(i - 1, "max_msgs"); - } - } - - // max_age - if (max_age > 0) { - const times = getTimes(); - if (times.length > 1) { - // expire messages older than max_age nanoseconds - // to avoid potential clock skew, we define *now* as the time of the most - // recent message. For us, this should be fine, since we only impose limits - // when writing new messages, and none of these limits are guaranteed. - const now = times[times.length - 1].time.valueOf(); - const cutoff = new Date(now - max_age); - for (let i = times.length - 2; i >= 0; i--) { - if (times[i].time < cutoff) { - // it just went over the limit. Everything before - // and including the i-th message should be deleted. - setIndex(i, "max_age"); - break; - } - } - } - } - - // max_bytes - if (max_bytes >= 0) { - let t = 0; - const times = getTimes(); - for (let i = times.length - 1; i >= 0; i--) { - t += this.sizes[times[i].key]; - if (t > max_bytes) { - // it just went over the limit. Everything before - // and including the i-th message must be deleted. - setIndex(i, "max_bytes"); - break; - } - } - } - - if (index > -1 && this.times != null) { - try { - // console.log("enforceLimits: deleting ", { index }); - const times = getTimes(); - const toDelete = times.slice(0, index + 1).map(({ key }) => key); - if (toDelete.length > 0) { - // console.log("enforceLImits: deleting ", toDelete.length, " keys"); - const revisions = { ...this.revisions }; - await awaitMap(toDelete, MAX_PARALLEL, async (key) => { - await this.delete(key, revisions[key]); - }); - } - } catch (err) { - // code 10071 is for "JetStreamApiError: wrong last sequence", which is - // expected when there are multiple clients, since all of them try to impose - // limits up at once. - if (err.code != "TIMEOUT" && err.code != 10071) { - console.log(`WARNING: expiring old messages - ${err}`); - } - } - } - }); - - // ensure any limits are satisfied, always by deleting old keys - private enforceLimits = throttle( - this.enforceLimitsNow, - ENFORCE_LIMITS_THROTTLE_MS, - { leading: false, trailing: true }, - ); -} - -// Support for value chunking below - -// **WARNING: Do not change these constants ever, or it will silently break -// all chunked kv and stream data that has ever been stored!!!** - -const CHUNK = "chunk"; -export const CHUNKS_HEADER = "CoCalc-Chunks"; - -function chunkedKey({ key, chunk }: { key: string; chunk?: number }) { - return `${key}.${chunk}.${CHUNK}`; -} - -function isChunkedKey(key: string) { - return key.endsWith("." + CHUNK); -} - -function getChunkCount(headers) { - if (headers == null) { - return 0; - } - for (const [key, value] of headers) { - if (key == CHUNKS_HEADER) { - return parseInt(value[0]); - } - } - return 0; -} - -function parseChunkedKey(key: string): { - key: string; - index: number; -} { - if (!isChunkedKey(key)) { - return { key, index: 0 }; - } - const v = key.split("."); - return { - key: v.slice(0, v.length - 2).join("."), - index: parseInt(v[v.length - 2]), - }; -} - -// The put function built into jetstream doesn't support -// setting headers, but we set headers for doing chunking. -// So we have to rewrite their put. I attempted to upstream this: -// https://github.com/nats-io/nats.js/issues/217 -// This was explicitly soundly rejected by the NATS developers. -// It's thus important that we unit test this, which is done in -// packages/backend/nats/test/sync/chunk.test.ts -// right now. I think it is highly unlikely NATS will break using -// headers in some future version, based on how KV is implemented -// on top of lower level primitives. However, if they do, we will -// fork whatever part of NATS that does, and maintain it. The code -// is easy to work with and understand. - -// Second, the put function in nats.js doesn't support setting a timeout, -// so that's another thing done below. Upstream: -// https://github.com/nats-io/nats.js/issues/268 -async function jetstreamPut( - kv, - k: string, - data, - opts: any = {}, -): Promise { - const ek = kv.encodeKey(k); - kv.validateKey(ek); - - const o = { timeout: opts.timeout } as any; - if (opts.previousSeq !== undefined) { - const h = createHeaders(); - o.headers = h; - // PubHeaders.ExpectedLastSubjectSequenceHdr is 'Nats-Expected-Last-Subject-Sequence', but - // PubHeaders is defined only internally to jetstream, so I copy/pasted this here. - h.set("Nats-Expected-Last-Subject-Sequence", `${opts.previousSeq}`); - } - if (opts.headers !== undefined) { - for (const [key, value] of opts.headers) { - if (o.headers == null) { - o.headers = createHeaders(); - } - o.headers.set(key, value[0]); - } - } - try { - await waitUntilConnected(); - const pa = await kv.js.publish(kv.subjectForKey(ek, true), data, o); - return pa.seq; - } catch (err) { - return Promise.reject(err); - } -} - -// see https://github.com/nats-io/nats.js/issues/246 -// In particular, we need this just to be able to support -// base64 encoded keys! - -// upstream is: /^[-/=.\w]+$/; - -const validKeyRe = /^[^\u0000\s*>]+$/; -function validateKey(k: string) { - if (k.startsWith(".") || k.endsWith(".") || !validKeyRe.test(k)) { - throw new Error(`invalid key: ${k}`); - } -} - -// upstream is: /^[-/=.>*\w]+$/; -const validSearchKey = /^[^\u0000\s]+$/; -export function validateSearchKey(k: string) { - if (k.startsWith(".") || k.endsWith(".") || !validSearchKey.test(k)) { - throw new Error(`invalid key: ${k}`); - } -} diff --git a/src/packages/nats/sync/inventory.ts b/src/packages/nats/sync/inventory.ts deleted file mode 100644 index dcb4c3da68..0000000000 --- a/src/packages/nats/sync/inventory.ts +++ /dev/null @@ -1,374 +0,0 @@ -/* -Inventory of all streams and key:value stores in a specific project, account or the public space. - -DEVELOPMENT: - -i = await require('@cocalc/backend/nats/sync').inventory({project_id:'00847397-d6a8-4cb0-96a8-6ef64ac3e6cf'}) - -i.ls() - -*/ - -import { dkv, type DKV } from "./dkv"; -import { dstream, type DStream } from "./dstream"; -import getTime from "@cocalc/nats/time"; -import refCache from "@cocalc/util/refcache"; -import type { JSONValue } from "@cocalc/util/types"; -import { - human_readable_size as humanReadableSize, - trunc_middle, -} from "@cocalc/util/misc"; -import type { ValueType } from "@cocalc/nats/types"; -import { type KVLimits } from "./general-kv"; -import { type FilteredStreamLimitOptions } from "./stream"; -import { DKO_PREFIX } from "./dko"; -import { waitUntilTimeAvailable } from "@cocalc/nats/time"; - -export const THROTTLE_MS = 10000; -export const INVENTORY_NAME = "CoCalc-Inventory"; - -type Sort = - | "last" - | "created" - | "count" - | "bytes" - | "name" - | "type" - | "valueType" - | "-last" - | "-created" - | "-count" - | "-bytes" - | "-name" - | "-type" - | "-valueType"; - -interface Location { - account_id?: string; - project_id?: string; -} - -type StoreType = "kv" | "stream"; - -interface Item { - // when it was created - created: number; - // last time this kv-store was updated - last: number; - // how much space is used by this kv-store - bytes: number; - // number of keys or messages - count: number; - // optional description, which can be anything - desc?: JSONValue; - // type of values stored - valueType?: ValueType; - // limits for purging old data - limits?: KVLimits | FilteredStreamLimitOptions; - // for streams, the seq number up to which this data is valid, i.e., - // this data is for all elements of the stream with sequence - // number <= seq. - seq?: number; -} - -interface FullItem extends Item { - type: StoreType; - name: string; -} - -export class Inventory { - public location: Location; - private dkv?: DKV; - - constructor(location: { account_id?: string; project_id?: string }) { - this.location = location; - } - - init = async () => { - this.dkv = await dkv({ - name: INVENTORY_NAME, - ...this.location, - }); - await waitUntilTimeAvailable(); - }; - - // Set but with NO LIMITS and no MERGE conflict algorithm. Use with care! - set = ({ - type, - name, - bytes, - count, - desc, - valueType, - limits, - seq, - }: { - type: StoreType; - name: string; - bytes: number; - count: number; - desc?: JSONValue; - valueType: ValueType; - limits?: KVLimits | FilteredStreamLimitOptions; - seq?: number; - }) => { - if (this.dkv == null) { - throw Error("not initialized"); - } - const last = getTime(); - const key = this.encodeKey({ name, type, valueType }); - const cur = this.dkv.get(key); - const created = cur?.created ?? last; - desc = desc ?? cur?.desc; - this.dkv.set(key, { - desc, - last, - created, - bytes, - count, - limits, - seq, - }); - }; - - private encodeKey = ({ name, type, valueType = "json" }) => - JSON.stringify({ name, type, valueType }); - - private decodeKey = (key) => JSON.parse(key); - - delete = ({ - name, - type, - valueType, - }: { - name: string; - type: StoreType; - valueType: ValueType; - }) => { - if (this.dkv == null) { - throw Error("not initialized"); - } - this.dkv.delete(this.encodeKey({ name, type, valueType })); - }; - - get = ( - x: { name: string; type: StoreType; valueType?: ValueType } | string, - ): (Item & { type: StoreType; name: string }) | undefined => { - if (this.dkv == null) { - throw Error("not initialized"); - } - let cur; - let name, type; - if (typeof x == "string") { - // just the name -- we infer/guess the type and valueType - name = x; - type = "kv"; - for (const valueType of ["json", "binary"]) { - cur = this.dkv.get(this.encodeKey({ name, type, valueType })); - if (cur == null) { - type = "stream"; - cur = this.dkv.get(this.encodeKey({ name, type, valueType })); - } - if (cur != null) { - break; - } - } - } else { - name = x.name; - cur = this.dkv.get(this.encodeKey(x)); - } - if (cur == null) { - return; - } - return { ...cur, type, name }; - }; - - getStores = async ({ filter }: { filter?: string } = {}): Promise< - (DKV | DStream)[] - > => { - const v: (DKV | DStream)[] = []; - for (const x of this.getAll({ filter })) { - const { desc, name, type } = x; - if (type == "kv") { - v.push(await dkv({ name, ...this.location, desc })); - } else if (type == "stream") { - v.push(await dstream({ name, ...this.location, desc })); - } else { - throw Error(`unknown store type '${type}'`); - } - } - return v; - }; - - needsUpdate = (x: { - name: string; - type: StoreType; - valueType: ValueType; - }): boolean => { - if (this.dkv == null) { - return false; - } - const cur = this.dkv.get(this.encodeKey(x)); - if (cur == null) { - return true; - } - // if (getTime() - cur.last >= 0.9 * THROTTLE_MS) { - // return true; - // } - return true; - }; - - getAll = ({ filter }: { filter?: string } = {}): FullItem[] => { - if (this.dkv == null) { - throw Error("not initialized"); - } - const all = this.dkv.getAll(); - if (filter) { - filter = filter.toLowerCase(); - } - const v: FullItem[] = []; - for (const key of Object.keys(all)) { - const { name, type, valueType } = this.decodeKey(key); - if (filter) { - const { desc } = all[key]; - const s = `${desc ? JSON.stringify(desc) : ""} ${name}`.toLowerCase(); - if (!s.includes(filter)) { - continue; - } - } - v.push({ ...all[key], name, type, valueType }); - } - return v; - }; - - close = async () => { - await this.dkv?.close(); - delete this.dkv; - }; - - private sortedKeys = (all, sort0: Sort) => { - let reverse: boolean, sort: string; - if (sort0[0] == "-") { - reverse = true; - sort = sort0.slice(1); - } else { - reverse = false; - sort = sort0; - } - // return keys of all, sorted as specified - const x: { k: string; v: any }[] = []; - for (const k in all) { - x.push({ k, v: { ...all[k], ...this.decodeKey(k) } }); - } - x.sort((a, b) => { - const a0 = a.v[sort]; - const b0 = b.v[sort]; - if (a0 < b0) { - return -1; - } - if (a0 > b0) { - return 1; - } - return 0; - }); - const y = x.map(({ k }) => k); - if (reverse) { - y.reverse(); - } - return y; - }; - - ls = ({ - log = console.log, - filter, - noTrunc, - path: path0, - sort = "-last", - }: { - log?: Function; - filter?: string; - noTrunc?: boolean; - path?: string; - sort?: Sort; - } = {}) => { - if (this.dkv == null) { - throw Error("not initialized"); - } - const all = this.dkv.getAll(); - log(` -Inventory for ${JSON.stringify(this.location)}`); - log( - "ls(opts: {filter?: string; noTrunc?: boolean; path?: string; sort?: 'last'|'created'|'count'|'bytes'|'name'|'type'|'valueType'|'-last'|...})", - ); - log( - "╭──────────┬─────────────────────────────────────────────────────┬───────────────────────┬──────────────────┬──────────────────┬──────────────────┬───────────────────────╮", - ); - log( - `│ ${padRight("Type", 7)} │ ${padRight("Name", 50)} │ ${padRight("Created", 20)} │ ${padRight("Size", 15)} │ ${padRight("Count", 15)} │ ${padRight("Value Type", 15)} │ ${padRight("Last Update", 20)} │`, - ); - log( - "├──────────┼─────────────────────────────────────────────────────┼───────────────────────┼──────────────────┼──────────────────┼──────────────────┼───────────────────────┤", - ); - for (const key of this.sortedKeys(all, sort)) { - const { last, created, count, bytes, desc, limits } = all[key]; - if (path0 && desc?.["path"] != path0) { - continue; - } - let { name, type, valueType } = this.decodeKey(key); - if (name.startsWith(DKO_PREFIX)) { - type = "kvobject"; - name = name.slice(DKO_PREFIX.length); - } - if (!noTrunc) { - name = trunc_middle(name, 50); - } - if ( - filter && - !`${desc ? JSON.stringify(desc) : ""} ${name}` - .toLowerCase() - .includes(filter.toLowerCase()) - ) { - continue; - } - log( - `│ ${padRight(type ?? "-", 7)} │ ${padRight(name, 50)} │ ${padRight(dateToString(new Date(created)), 20)} │ ${padRight(humanReadableSize(bytes), 15)} │ ${padRight(count, 15)} │ ${padRight(valueType, 15)} │ ${padRight(dateToString(new Date(last)), 20)} │`, - ); - if (desc) { - log(`│ | ${padRight(JSON.stringify(desc), 153)} |`); - } - if (limits) { - log(`│ │ ${padRight(JSON.stringify(limits), 153)} |`); - } - } - log( - "╰──────────┴─────────────────────────────────────────────────────┴───────────────────────┴──────────────────┴──────────────────┴──────────────────┴───────────────────────╯", - ); - }; -} - -function dateToString(d: Date) { - return d.toISOString().replace("T", " ").replace("Z", "").split(".")[0]; -} - -function padRight(s: any, n) { - if (typeof s != "string") { - s = `${s}`; - } - while (s.length <= n) { - s += " "; - } - return s; -} - -export const cache = refCache({ - name: "inventory", - createObject: async (loc) => { - const k = new Inventory(loc); - await k.init(); - return k; - }, -}); - -export async function inventory(options: Location = {}): Promise { - return await cache(options); -} diff --git a/src/packages/nats/sync/kv.ts b/src/packages/nats/sync/kv.ts deleted file mode 100644 index 62ce949bc0..0000000000 --- a/src/packages/nats/sync/kv.ts +++ /dev/null @@ -1,186 +0,0 @@ -/* -Async Consistent Centralized Key Value Store - -NOTE: I think this isn't used by anything actually. Note it doesn't emit -change events. Maybe we should delete this? - -DEVELOPMENT: - -~/cocalc/src/packages/backend$ n -Welcome to Node.js v18.17.1. -Type ".help" for more information. -> t = await require("@cocalc/backend/nats/sync").kv({name:'test'}) - -*/ - -import { EventEmitter } from "events"; -import { type NatsEnv, type Location } from "@cocalc/nats/types"; -import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; -import { GeneralKV, type KVLimits } from "./general-kv"; -import { jsName, localLocationName } from "@cocalc/nats/names"; -import refCache from "@cocalc/util/refcache"; -import { getEnv } from "@cocalc/nats/client"; -import type { JSONValue } from "@cocalc/util/types"; -import type { ValueType } from "@cocalc/nats/types"; -import { decodeBase64, encodeBase64 } from "@cocalc/nats/util"; -import jsonStableStringify from "json-stable-stringify"; - -export interface KVOptions extends Location { - name: string; - env?: NatsEnv; - limits?: Partial; - noCache?: boolean; - desc?: JSONValue; - valueType?: ValueType; -} - -export class KV extends EventEmitter { - generalKV?: GeneralKV; - name: string; - private prefix: string; - - constructor(options: KVOptions) { - super(); - const { name, account_id, project_id, env, limits, valueType } = options; - // name of the jetstream key:value store. - const kvname = jsName({ account_id, project_id }); - this.name = name + localLocationName(options); - if (env == null) { - throw Error("env must be defined"); - } - this.prefix = encodeBase64(this.name); - this.generalKV = new GeneralKV({ - name: kvname, - filter: `${this.prefix}.>`, - env, - limits, - valueType, - }); - this.init(); - return new Proxy(this, { - deleteProperty(target, prop) { - if (typeof prop == "string") { - target.delete(prop); - return true; - } else { - return false; - } - }, - set(target, prop, value) { - prop = String(prop); - if (prop == "_eventsCount" || prop == "_events" || prop == "close") { - target[prop] = value; - return true; - } - if (target[prop] != null) { - throw Error(`method name '${prop}' is read only`); - } - target.set(prop, value); - return true; - }, - get(target, prop) { - return target[String(prop)] ?? target.get(String(prop)); - }, - }); - } - - init = reuseInFlight(async () => { - if (this.generalKV == null) { - throw Error("closed"); - } - await this.generalKV.init(); - }); - - close = () => { - if (this.generalKV == null) { - return; - } - this.generalKV.close(); - delete this.generalKV; - this.emit("closed"); - this.removeAllListeners(); - }; - - delete = async (key: string) => { - if (this.generalKV == null) { - throw Error("closed"); - } - await this.generalKV.delete(`${this.prefix}.${encodeBase64(key)}`); - }; - - // delete everything - clear = async () => { - if (this.generalKV == null) { - throw Error("closed"); - } - await this.generalKV.clear(); - }; - - // server assigned time - time = (key?: string): { [key: string]: Date } | Date | undefined => { - if (this.generalKV == null) { - throw Error("closed"); - } - return this.generalKV.time( - key ? `${this.prefix}.${encodeBase64(key)}` : undefined, - ); - }; - - get = (key: string): T | undefined => { - if (this.generalKV == null) { - throw Error("closed"); - } - return this.generalKV.get(`${this.prefix}.${encodeBase64(key)}`); - }; - - getAll = (): { [key: string]: T } => { - if (this.generalKV == null) { - throw Error("closed"); - } - const obj = this.generalKV.getAll(); - const x: any = {}; - for (const k in obj) { - const h = this.generalKV.headers(k); - if (h?.key == null) { - throw Error(`missing header for key ${k}`); - } - const key = decodeBase64(h.key); - x[key] = obj[k]; - } - return x; - }; - - set = async (key: string, value: T) => { - if (this.generalKV == null) { - throw Error("closed"); - } - await this.generalKV.set(`${this.prefix}.${encodeBase64(key)}`, value, { - headers: { key: encodeBase64(key) }, - }); - }; -} - -export function userKvKey(options: KVOptions) { - if (!options.name) { - throw Error("name must be specified"); - } - const { env, ...x } = options; - return jsonStableStringify(x); -} - -export const cache = refCache({ - name: "kv", - createKey: userKvKey, - createObject: async (opts) => { - if (opts.env == null) { - opts.env = await getEnv(); - } - const k = new KV(opts); - await k.init(); - return k; - }, -}); - -export async function kv(options: KVOptions): Promise> { - return await cache(options); -} diff --git a/src/packages/nats/sync/stream.ts b/src/packages/nats/sync/stream.ts deleted file mode 100644 index 027ac0d681..0000000000 --- a/src/packages/nats/sync/stream.ts +++ /dev/null @@ -1,1111 +0,0 @@ -/* -Consistent Centralized Event Stream = ordered list of messages - -DEVELOPMENT: - -# note the package directory!! -~/cocalc/src/packages/backend n -Welcome to Node.js v18.17.1. -Type ".help" for more information. -> s = await require("@cocalc/backend/nats/sync").stream({name:'test'}) - - -> env = await require("@cocalc/backend/nats/env").getEnv(); a = require("@cocalc/nats/sync/stream"); s = new a.Stream({name:'test',env,subjects:'foo',filter:'foo'}); await s.init(); - - -With browser client using a project: - -# in browser -> s = await cc.client.nats_client.stream({project_id:cc.current().project_id,name:'foo'}) - -# in node: -> env = await require("@cocalc/backend/nats/env").getEnv(); a = require("@cocalc/nats/sync/stream"); s = await a.stream({project_id:cc.current().project_id,name:'foo', env}) - - -# Involving limits: - -> env = await require("@cocalc/backend/nats/env").getEnv(); a = require("@cocalc/nats/sync/stream"); s = await a.stream({project_id:cc.current().project_id,name:'foo', env, limits:{max_msgs:5,max_age:1000*15,max_bytes:10000,max_msg_size:1000}}) -> s.getAll() - -In browser: -> s = await cc.client.nats_client.stream({project_id:cc.current().project_id, name:'foo',limits:{max_msgs:5,max_age:1000*15,max_bytes:10000,max_msg_size:1000}}) - -TODO: - - maybe the limits and other config should be stored in a KV store so - they are sync'd between clients automatically. That's what NATS surely - does internally. - - -*/ - -import { EventEmitter } from "events"; -import { type NatsEnv, type ValueType } from "@cocalc/nats/types"; -import { - jetstreamManager, - jetstream, - type JetStreamPublishOptions, - AckPolicy, -} from "@nats-io/jetstream"; -import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; -import { jsName, streamSubject } from "@cocalc/nats/names"; -import { - getMaxPayload, - waitUntilConnected, - isConnected, - millis, - encodeBase64, - nanos, -} from "@cocalc/nats/util"; -import { delay } from "awaiting"; -import { throttle } from "lodash"; -import { isNumericString } from "@cocalc/util/misc"; -import { map as awaitMap } from "awaiting"; -import refCache from "@cocalc/util/refcache"; -import { type JsMsg } from "@nats-io/jetstream"; -import { getEnv } from "@cocalc/nats/client"; -import type { JSONValue } from "@cocalc/util/types"; -import { headers as createHeaders } from "@nats-io/nats-core"; -import { CHUNKS_HEADER } from "./general-kv"; -import jsonStableStringify from "json-stable-stringify"; -import { asyncDebounce } from "@cocalc/util/async-utils"; -import { waitUntilReady } from "@cocalc/nats/tiered-storage/client"; -import { COCALC_MESSAGE_ID_HEADER, type RawMsg } from "./ephemeral-stream"; - -const PUBLISH_TIMEOUT = 15000; - -class PublishRejectError extends Error { - code: string; - mesg: any; - subject?: string; - limit?: string; -} - -const MAX_PARALLEL = 50; - -const CONNECTION_CHECK_INTERVAL = 5000; - -// Making this too LONG is very dangerous since it increases load on the server. -// Making it too short means it has to get recreated whenever the network connection drops. -const EPHEMERAL_CONSUMER_THRESH = 45 * 1000; - -//console.log("!!! ALERT: USING VERY SHORT CONSUMERS FOR TESTING!"); -//const EPHEMERAL_CONSUMER_THRESH = 3 * 1000; - -// We re-implement exactly the same stream-wide limits that NATS has, -// but instead, these are for the stream **with the given filter**. -// Limits are enforced by all clients *client side* within ENFORCE_LIMITS_THROTTLE_MS -// of any client making changes. It is important to significantly throttle -// this, as it can be expensive to the server. -// Also, obviously the true limit is the minimum of the full NATS stream limits and -// these limits. - -// Significant throttling is VERY, VERY important, since purging old messages frequently -// seems to put a very significant load on NATS! -export const ENFORCE_LIMITS_THROTTLE_MS = process.env.COCALC_TEST_MODE - ? 100 - : 45000; - -export interface FilteredStreamLimitOptions { - // How many messages may be in a Stream, oldest messages will be removed - // if the Stream exceeds this size. -1 for unlimited. - max_msgs: number; - // Maximum age of any message in the stream matching the filter, - // expressed in milliseconds. 0 for unlimited. - // **Note that max_age is in milliseoncds, NOT nanoseconds like in Nats!!!** - max_age: number; - // How big the Stream may be, when the combined stream size matching the filter - // exceeds this old messages are removed. -1 for unlimited. - // This is enforced only on write, so if you change it, it only applies - // to future messages. - max_bytes: number; - // The largest message that will be accepted by the Stream. -1 for unlimited. - max_msg_size: number; - - // Attempting to publish a message that causes this to be exceeded - // throws an exception instead. -1 (or 0) for unlimited - // For dstream, the messages are explicitly rejected and the client - // gets a "reject" event emitted. E.g., the terminal running in the project - // writes [...] when it gets these rejects, indicating that data was - // dropped. - max_bytes_per_second: number; - max_msgs_per_second: number; -} - -export interface StreamOptions { - // what it's called by us - name: string; - // actually name of the jetstream in NATS - jsname: string; - // subject = default subject used for publishing; defaults to filter if filter doesn't have any wildcard - subject?: string; - subjects: string | string[]; - filter?: string; - env?: NatsEnv; - natsStreamOptions?; - limits?: Partial; - // only load historic messages starting at the given seq number. - start_seq?: number; - desc?: JSONValue; - valueType?: ValueType; -} - -export class Stream extends EventEmitter { - public readonly name: string; - public readonly jsname: string; - private natsStreamOptions?; - private limits: FilteredStreamLimitOptions; - private bytesSent: { [time: number]: number } = {}; - private subjects: string | string[]; - private filter?: string; - private subject?: string; - private env: NatsEnv; - private _start_seq?: number; - private js; - private jsm; - private stream?; - private watch?; - public readonly valueType: ValueType; - // seq = the last sequence number of a message on this stream that we received - // from NATS. This is used only for resuming without missing anything. - private last_seq: number = 1; - - // don't do "this.raw=" or "this.messages=" anywhere in this class!!! - public readonly raw: JsMsg[][] = []; - public readonly messages: T[] = []; - private consumer?; - - constructor({ - name, - jsname, - env, - subject, - subjects, - filter, - natsStreamOptions, - limits, - start_seq, - valueType = "json", - }: StreamOptions) { - super(); - - this.valueType = valueType; - if (env == null) { - throw Error("bug: env must be specified"); - } - this.env = env; - // create a jetstream client so we can publish to the stream - this.js = jetstream(env.nc); - this.name = name; - this.jsname = jsname; - this.natsStreamOptions = natsStreamOptions; - if ( - subject == null && - filter != null && - !filter.includes("*") && - !filter.includes(">") - ) { - subject = filter; - } - this.subject = subject; - this.subjects = typeof subjects == "string" ? [subjects] : subjects; - if (this.subjects.length == 0) { - throw Error("subjects must be at least one string"); - } - this.filter = filter; - this._start_seq = start_seq; - this.limits = { - max_msgs: -1, - max_age: 0, - max_bytes: -1, - max_msg_size: -1, - max_bytes_per_second: -1, - max_msgs_per_second: -1, - ...limits, - }; - return new Proxy(this, { - get(target, prop) { - return typeof prop == "string" && isNumericString(prop) - ? target.get(parseInt(prop)) - : target[String(prop)]; - }, - }); - } - - init = reuseInFlight(async () => { - if (this.stream != null) { - return; - } - this.jsm = await jetstreamManager(this.env.nc); - const options = { - subjects: this.subjects, - compression: "s2", - // our streams are relatively small so a longer duplicate window than 2 minutes seems ok. - duplicate_window: nanos(1000 * 60 * 5), - ...this.natsStreamOptions, - }; - await waitUntilReady(this.jsname); - try { - this.stream = await this.jsm.streams.add({ - name: this.jsname, - ...options, - }); - } catch (err) { - // probably already exists, so try to modify to have the requested properties. - this.stream = await this.jsm.streams.update(this.jsname, options); - } - await this.fetchInitialData({ - // do not broadcast initial load - noEmit: true, - }); - if (this.stream == null) { - // closed *during* initial load - return; - } - this.ensureConnected(); - this.watchForNewData(); - }); - - private ensureConnected = async () => { - if (this.env.nc.on != null) { - this.env.nc.on("reconnect", this.restartConsumer); - this.env.nc.on("status", ({ type }) => { - if (type == "reconnect") { - this.ensureConsumerIsValid(); - } - }); - } else { - this.checkConsumerOnReconnect(); - } - while (this.stream != null) { - if (!(await isConnected())) { - await this.restartConsumer(); - } - await delay(CONNECTION_CHECK_INTERVAL); - } - }; - - // We can't do this all the time due to efficiency - // (see https://www.synadia.com/blog/jetstream-design-patterns-for-scale) - // but we **MUST do it around connection events** - // no matter what the docs say or otherwise!!!! - // At least with the current nats.js drivers. - // Often nats does recreate the consumer, but sometimes it doesn't. - // I can't nail down which is which. - private ensureConsumerIsValid = asyncDebounce( - async () => { - await waitUntilConnected(); - await delay(2000); - const isValid = await this.isConsumerStillValid(); - if (!isValid) { - if (this.stream == null) { - return; - } - console.log( - `nats stream: ${this.name} -- consumer not valid, so recreating`, - ); - await this.restartConsumer(); - } - }, - 3000, - { leading: false, trailing: true }, - ); - - private checkConsumerOnReconnect = async () => { - while (this.stream != null) { - try { - for await (const { type } of await this.env.nc.status()) { - if (type == "reconnect") { - await this.ensureConsumerIsValid(); - } - } - } catch { - await delay(15000); - await this.ensureConsumerIsValid(); - } - } - }; - - private isConsumerStillValid = async () => { - await waitUntilConnected(); - if (this.consumer == null || this.stream == null) { - return false; - } - try { - await this.consumer.info(); - return true; - } catch (err) { - console.log(`nats: consumer.info error -- ${err}`); - return false; - } - }; - - get = (n?): T | T[] => { - if (this.js == null) { - throw Error("closed"); - } - if (n == null) { - return this.getAll(); - } else { - return this.messages[n]; - } - }; - - getAll = (): T[] => { - if (this.js == null) { - throw Error("closed"); - } - return [...this.messages]; - }; - - headers = (n: number): { [key: string]: string } | undefined => { - return headersFromRawMessages(this.raw[n]); - }; - - // get server assigned global sequence number of n-th message in stream - seq = (n: number): number | undefined => { - return last(this.raw[n])?.seq; - }; - - // get server assigned time of n-th message in stream - time = (n: number): Date | undefined => { - const r = last(this.raw[n]); - if (r == null) { - return; - } - return new Date(millis(r?.info.timestampNanos)); - }; - - times = (): (Date | undefined)[] => { - const v: (Date | undefined)[] = []; - for (let i = 0; i < this.length; i++) { - v.push(this.time(i)); - } - return v; - }; - - get length(): number { - return this.messages.length; - } - - get start_seq(): number | undefined { - return this._start_seq; - } - - // WARNING: if you push multiple values at once here, then the order is NOT guaranteed - push = async (...args: T[]) => { - await awaitMap(args, MAX_PARALLEL, this.publish); - }; - - private encodeValue = (value) => { - return this.valueType == "json" ? this.env.jc.encode(value) : value; - }; - - private decodeValue = (value) => { - return this.valueType == "json" ? this.env.jc.decode(value) : value; - }; - - publish = async ( - mesg: T, - options?: Partial< - JetStreamPublishOptions & { headers: { [key: string]: string } } - >, - ) => { - if (this.js == null) { - throw Error("closed"); - } - const data = this.encodeValue(mesg); - if ( - this.limits.max_msg_size > -1 && - data.length > this.limits.max_msg_size - ) { - const err = new PublishRejectError( - `message size (=${data.length}) exceeds max_msg_size=${this.limits.max_msg_size} bytes`, - ); - err.code = "REJECT"; - err.mesg = mesg; - err.subject = this.subject; - err.limit = "max_msg_size"; - throw err; - } - this.enforceLimits(); - if (options?.msgID) { - if (options.headers) { - // also put it here so can be used to clear this.local by dstream: - options.headers[COCALC_MESSAGE_ID_HEADER] = options.msgID; - } else { - options.headers = { [COCALC_MESSAGE_ID_HEADER]: options.msgID }; - } - } - let resp; - const chunks: Buffer[] = []; - const headers: ReturnType[] = []; - // we subtract off from max_payload to leave space for headers (technically, 10 is enough) - await waitUntilConnected(); - const maxMessageSize = (await getMaxPayload()) - 1000; - //const maxMessageSize = 20; // DEV ONLY!!! - - // this may throw an exception: - enforceRateLimits({ - limits: this.limits, - bytesSent: this.bytesSent, - subject: this.subject, - data, - mesg, - }); - - if (data.length > maxMessageSize) { - // we chunk the message into blocks of size maxMessageSize, - // to fit NATS message size limits. We include a header - // so we can re-assemble the chunks later. - let data0 = data; - while (data0.length > 0) { - chunks.push(data0.slice(0, maxMessageSize)); - data0 = data0.slice(maxMessageSize); - } - const last = chunks.length; - for (let i = 1; i <= last; i++) { - const h = createHeaders(); - if (i == 1 && options?.headers != null) { - // also include custom user headers - for (const k in options.headers) { - h.append(k, `${options.headers[k]}`); - } - } - h.append(CHUNKS_HEADER, `${i}/${last}`); - headers.push(h); - } - } else { - // trivial chunk and no header needed. - chunks.push(data); - if (options?.headers != null) { - const h = createHeaders(); - for (const k in options.headers) { - h.append(k, `${options.headers[k]}`); - } - headers.push(h); - } - } - - for (let i = 0; i < chunks.length; i++) { - try { - await waitUntilConnected(); - resp = await this.js.publish(this.subject, chunks[i], { - timeout: PUBLISH_TIMEOUT, - ...options, - // if options contains a msgID, we must make it different for each chunk; - // otherwise, all but the first chunk is discarded! - ...(options?.msgID == null - ? undefined - : { msgID: `${options.msgID}-${i}` }), - headers: headers[i], - }); - // NOTE: the resp we get back contains a sequence number and GUARANTEES that the - // data has been written to disk by the nats server. - } catch (err) { - if (err.code == "MAX_PAYLOAD_EXCEEDED") { - // nats rejects due to payload size - const err2 = new PublishRejectError(`${err}`); - err2.code = "REJECT"; - err2.mesg = mesg; - err2.subject = this.subject; - throw err2; - } else { - throw err; - } - } - } - this.enforceLimits(); - return resp; - }; - - private getConsumer = async ({ start_seq }: { start_seq?: number } = {}) => { - // NOTE: do not cache or modify this in this function getConsumer, - // since it is also called by load and when reconnecting. - const js = jetstream(this.env.nc); - const jsm = await jetstreamManager(this.env.nc); - // making an ephemeral consumer, which is automatically destroyed by NATS - // after inactive_threshold. At that point we MUST reset state. - const options = { - filter_subject: this.filter, - ack_policy: AckPolicy.Explicit, - inactive_threshold: nanos(EPHEMERAL_CONSUMER_THRESH), - }; - let startOptions; - if (start_seq == null && this._start_seq != null) { - start_seq = this._start_seq; - } - if (start_seq != null) { - startOptions = { - deliver_policy: "by_start_sequence", - opt_start_seq: start_seq, - }; - } else { - startOptions = {}; - } - const { name } = await jsm.consumers.add(this.jsname, { - ...options, - ...startOptions, - }); - if (this.consumer != null) { - try { - await this.consumer.delete(); - } catch { - // this absolutely *can* throw an error if the consumer was already deleted - // automatically on the server for some reason! - } - delete this.consumer; - } - this.consumer = await js.consumers.get(this.jsname, name); - return this.consumer; - }; - - private fetchInitialData = async ({ - options, - noEmit, - }: { - options?; - noEmit: boolean; - }) => { - const consumer = await this.getConsumer(options); - // grab the messages. This should be very efficient since it - // internally grabs them in batches. - // This code seems exactly necessary and efficient, and most - // other things I tried ended too soon or hung. See also - // comment in getAllFromKv about permissions. - while (true) { - // https://www.synadia.com/blog/jetstream-design-patterns-for-scale says - // "Consumer info is also frequently misused as a method for clients to check - // for pending messages. Instead, get this metadata from the last message - // fetched to avoid the unnecessary overhead of consumer info." - const info = await consumer.info(); - if (info.num_pending == 0) { - return consumer; - } - const fetch = await consumer.fetch({ max_messages: 1000 }); - this.watch = fetch; - let chunks: JsMsg[] = []; - for await (const mesg of fetch) { - mesg.ack(); - let isChunked = false; - // chunked? - if (mesg.headers != null) { - for (const [key, value] of mesg.headers) { - if (key == CHUNKS_HEADER) { - isChunked = true; - const v = value[0].split("/"); - if (v[0] == "1") { - // first chunk - chunks = [mesg]; - } else { - chunks.push(mesg); - } - if (v[0] == v[1]) { - // have all the chunks - this.handle(chunks, noEmit); - this.enforceLimits(); - } - } - } - } - if (!isChunked) { - // not chunked - this.handle([mesg], noEmit); - this.enforceLimits(); - } - const pending = mesg.info.pending; - if (pending <= 0) { - return consumer; - } - } - } - }; - - private watchForNewData = async () => { - if (this.stream == null) { - // closed *during* initial load - return; - } - // STAGE 2: Watch for new mesg. It's the same consumer though, - // so we are **guaranteed** not to miss anything. - this.enforceLimits(); - this.emit("connected"); - const consume = await this.consumer.consume(); - this.watch = consume; - let chunks: JsMsg[] = []; - for await (const mesg of consume) { - mesg.ack(); - let isChunked = false; - // chunked? - for (const [key, value] of mesg.headers ?? []) { - if (key == CHUNKS_HEADER) { - isChunked = true; - const v = value[0].split("/"); - if (v[0] == "1") { - // first chunk - chunks = [mesg]; - } else { - chunks.push(mesg); - } - if (v[0] == v[1]) { - // have all the chunks - this.handle(chunks, false); - this.enforceLimits(); - } - } - } - if (!isChunked) { - // not chunked - this.handle([mesg], false); - this.enforceLimits(); - } - } - }; - - // this does not throw an exception -- it keeps trying until success. - private restartConsumer = reuseInFlight(async (): Promise => { - await waitUntilConnected(); - if (this.stream == null) { - return; - } - // make a new consumer, starting AFTER the last event we retrieved - let d = 250; - while (true) { - this.watch?.stop(); // stop current watch (if any) - // make new one: - const start_seq = this.last_seq + 1; - try { - // noEmit = false since we DO want to emit an event for any changes at this point!! - this.consumer = await this.fetchInitialData({ - options: { start_seq }, - noEmit: false, - }); - if (this.stream == null) { - // closed - return; - } - this.watchForNewData(); - return; - } catch (err) { - d = Math.min(30000, d * 1.3) + Math.random(); - await delay(d); - } - } - }); - - private decode = (raw: JsMsg[]) => { - if (raw.length == 0) { - throw Error("must be at least one chunk"); - } - const data = - raw.length == 1 - ? raw[0].data - : // @ts-ignore -- for nextjs prod - Buffer.concat(raw.map((mesg) => mesg.data)); - - try { - return this.decodeValue(data); - } catch (_err) { - // console.log("WARNING: issue decoding nats stream data", { data, _err }); - // better than crashing: - return data; - } - }; - - private handle = (raw: JsMsg[], noEmit: boolean) => { - const mesg = this.decode(raw); - this.messages.push(mesg); - this.raw.push(raw); - for (const { seq } of raw) { - this.last_seq = Math.max(this.last_seq, seq); - } - if (!noEmit) { - this.emit("change", mesg, raw); - } - }; - - close = () => { - if (this.watch == null) { - return; - } - (async () => { - try { - await this.consumer?.delete(); - delete this.consumer; - } catch { - // this absolutely *can* throw an error if the consumer was already deleted - // automatically on the server for some reason! - } - })(); - this.watch.stop(); - delete this.watch; - delete this.stream; - delete this.jsm; - delete this.js; - this.emit("closed"); - this.removeAllListeners(); - this.env.nc.removeListener?.("reconnect", this.restartConsumer); - }; - - // delete all messages up to and including the - // one at position index, i.e., this.messages[index] - // is deleted. - // NOTE: other clients will NOT see the result of a purge, - // except when done implicitly via limits, since all clients - // truncate this.raw and this.messages directly. - purge = async ({ index = -1 }: { index?: number } = {}) => { - // console.log("purge", { index }); - if (index >= this.raw.length - 1 || index == -1) { - index = this.raw.length - 1; - // everything - // console.log("purge everything"); - await this.jsm.streams.purge(this.jsname, { - filter: this.filter, - }); - } else { - const { seq } = last(this.raw[index + 1]); - await this.jsm.streams.purge(this.jsname, { - filter: this.filter, - seq, - }); - } - this.messages.splice(0, index + 1); - this.raw.splice(0, index + 1); - }; - - // get stats for this stream using data we have already downloaded, BUT - // only considering messages with sequence >= start_seq. - stats = ({ - start_seq = 1, - }: { - start_seq?: number; - }): { count: number; bytes: number } | undefined => { - if (this.raw == null) { - return; - } - let count = 0; - let bytes = 0; - for (const raw of this.raw) { - const seq = last(raw)?.seq; - if (seq == null) { - continue; - } - if (seq < start_seq) { - continue; - } - count += 1; - for (const r of raw) { - bytes += r.data.length; - } - } - return { count, bytes }; - }; - - private enforceLimitsNow = reuseInFlight(async () => { - if (this.jsm == null) { - return; - } - const index = enforceLimits({ - messages: this.messages, - raw: this.raw, - limits: this.limits, - }); - // console.log("enforceLImits", { index }); - if (index > -1) { - try { - // console.log("imposing limit via purge ", { index }); - await this.purge({ index }); - } catch (err) { - if (err.code != "TIMEOUT") { - console.log(`WARNING: purging old messages - ${err}`); - } - } - } - }); - - // ensure any limits are satisfied, i.e., delete old messages. - private enforceLimits = throttle( - this.enforceLimitsNow, - ENFORCE_LIMITS_THROTTLE_MS, - { leading: false, trailing: true }, - ); - - // load older messages starting at start_seq - load = async ({ - start_seq, - noEmit, - }: { - start_seq: number; - noEmit?: boolean; - }) => { - if (this._start_seq == null || this._start_seq <= 1) { - // we already loaded everything on initialization; there can't be anything older. - return; - } - const consumer = await this.getConsumer({ start_seq }); - // https://www.synadia.com/blog/jetstream-design-patterns-for-scale says - // "Consumer info is also frequently misused as a method for clients to check - // for pending messages. Instead, get this metadata from the last message - // fetched to avoid the unnecessary overhead of consumer info." - const info = await consumer.info(); - const fetch = await consumer.fetch(); - let i = 0; - // grab the messages. This should be very efficient since it - // internally grabs them in batches. - const raw: JsMsg[][] = []; - const messages: T[] = []; - const cur = last(this.raw[0])?.seq; - let chunks: JsMsg[] = []; - for await (const mesg of fetch) { - if (cur != null && mesg.seq >= cur) { - break; - } - - let isChunked = false; - // chunked? - for (const [key, value] of mesg.headers ?? []) { - if (key == CHUNKS_HEADER) { - isChunked = true; - const v = value[0].split("/"); - if (v[0] == "0") { - // first chunk - chunks = [mesg]; - } else { - chunks.push(mesg); - } - if (v[0] == v[1]) { - // have all the chunks - raw.push(chunks); - messages.push(this.decodeValue(chunks)); - } - } - } - if (!isChunked) { - // not chunked - raw.push([mesg]); - messages.push(this.decode([mesg])); - } - i += 1; - if (i >= info.num_pending) { - break; - } - } - // mutate the array this.raw and this.messages by splicing in - // raw and messages at the beginning: - this.raw.unshift(...raw); - this.messages.unshift(...messages); - if (!noEmit) { - for (let i = 0; i < raw.length; i++) { - this.emit("change", messages[i], raw[i]); - } - } - this._start_seq = start_seq; - }; -} - -// One stream for each account and one for each project. -// Use the filters to restrict, e.g., to message about a particular file. - -export interface UserStreamOptions { - name: string; - env?: NatsEnv; - account_id?: string; - project_id?: string; - limits?: Partial; - start_seq?: number; - noCache?: boolean; - desc?: JSONValue; - valueType?: ValueType; -} - -export function userStreamOptionsKey(options: UserStreamOptions) { - if (!options.name) { - throw Error("name must be specified"); - } - const { env, ...x } = options; - return jsonStableStringify(x); -} - -export const cache = refCache({ - name: "stream", - createKey: userStreamOptionsKey, - createObject: async (options) => { - if (options.env == null) { - options.env = await getEnv(); - } - const { account_id, project_id, name } = options; - const jsname = jsName({ account_id, project_id }); - const subjects = streamSubject({ account_id, project_id }); - const filter = subjects.replace(">", encodeBase64(name)); - const stream = new Stream({ - ...options, - name, - jsname, - subjects, - subject: filter, - filter, - }); - await stream.init(); - return stream; - }, -}); - -export async function stream( - options: UserStreamOptions, -): Promise> { - return await cache(options); -} - -export function last(v: any[] | undefined) { - if (v === undefined) { - return v; - } - return v[v.length - 1]; -} - -export function enforceLimits({ - messages, - raw, - limits, -}: { - messages: any[]; - raw: (JsMsg | RawMsg)[][]; - limits: FilteredStreamLimitOptions; -}) { - const { max_msgs, max_age, max_bytes } = limits; - // we check with each defined limit if some old messages - // should be dropped, and if so move limit forward. If - // it is above -1 at the end, we do the drop. - let index = -1; - const setIndex = (i, _limit) => { - // console.log("setIndex", { i, _limit }); - index = Math.max(i, index); - }; - // max_msgs - // console.log({ max_msgs, l: messages.length, messages }); - if (max_msgs > -1 && messages.length > max_msgs) { - // ensure there are at most limits.max_msgs messages - // by deleting the oldest ones up to a specified point. - const i = messages.length - max_msgs; - if (i > 0) { - setIndex(i - 1, "max_msgs"); - } - } - - // max_age - if (max_age > 0) { - // expire messages older than max_age nanoseconds - const recent = raw[raw.length - 1]; - if (recent != null) { - // to avoid potential clock skew, we define *now* as the time of the most - // recent message. For us, this should be fine, since we only impose limits - // when writing new messages, and none of these limits are guaranteed. - const nanos = last(recent).info?.timestampNanos; - const now = nanos ? nanos / 10 ** 6 : last(recent).timestamp; - if (now) { - const cutoff = now - max_age; - for (let i = raw.length - 1; i >= 0; i--) { - const nanos = last(raw[i]).info?.timestampNanos; - const t = nanos ? nanos / 10 ** 6 : last(raw[i]).timestamp; - if (t < cutoff) { - // it just went over the limit. Everything before - // and including the i-th message must be deleted. - setIndex(i, "max_age"); - break; - } - } - } - } - } - - // max_bytes - if (max_bytes >= 0) { - let t = 0; - for (let i = raw.length - 1; i >= 0; i--) { - for (const r of raw[i]) { - t += r.data.length; - } - if (t > max_bytes) { - // it just went over the limit. Everything before - // and including the i-th message must be deleted. - setIndex(i, "max_bytes"); - break; - } - } - } - - return index; -} - -export function enforceRateLimits({ - limits, - bytesSent, - subject, - data, - mesg, -}: { - limits: { max_bytes_per_second: number; max_msgs_per_second: number }; - bytesSent: { [time: number]: number }; - subject?: string; - data; - mesg; -}) { - const now = Date.now(); - if (!(limits.max_bytes_per_second > 0) && !(limits.max_msgs_per_second > 0)) { - return; - } - - const cutoff = now - 1000; - let bytes = 0, - msgs = 0; - for (const t in bytesSent) { - if (parseInt(t) < cutoff) { - delete bytesSent[t]; - } else { - bytes += bytesSent[t]; - msgs += 1; - } - } - if ( - limits.max_bytes_per_second > 0 && - bytes + data.length > limits.max_bytes_per_second - ) { - const err = new PublishRejectError( - `bytes per second limit of ${limits.max_bytes_per_second} exceeded`, - ); - err.code = "REJECT"; - err.mesg = mesg; - err.subject = subject; - err.limit = "max_bytes_per_second"; - throw err; - } - if (limits.max_msgs_per_second > 0 && msgs > limits.max_msgs_per_second) { - const err = new PublishRejectError( - `messages per second limit of ${limits.max_msgs_per_second} exceeded`, - ); - err.code = "REJECT"; - err.mesg = mesg; - err.subject = subject; - err.limit = "max_msgs_per_second"; - throw err; - } - bytesSent[now] = data.length; -} - -export function headersFromRawMessages(messages?: (JsMsg | RawMsg)[]) { - if (messages == null) { - return undefined; - } - const x: { [key: string]: string } = {}; - let hasHeaders = false; - for (const raw of messages) { - const { headers } = raw; - if (headers == null) { - continue; - } - for (const [key, value] of headers) { - x[key] = value[0]; - hasHeaders = true; - } - } - return hasHeaders ? x : undefined; -} diff --git a/src/packages/nats/system.ts b/src/packages/nats/system.ts deleted file mode 100644 index 73b7b94a6f..0000000000 --- a/src/packages/nats/system.ts +++ /dev/null @@ -1,28 +0,0 @@ -/* -This is a key:value store that hubs can write to and all -users of cocalc can read from. It contains: - -- recent system-wide notifications that haven't been canceled - system.notifications.{random} - -- the customize data: what used to be the /customize http endpoint - this makes it so clients get notified whenever anything changes, e.g., when the - recommended or required version changes, and can act accordingly. The UI - can also change. - -Development: - -~/cocalc/src/packages/server$ n -Welcome to Node.js v18.17.1. -Type ".help" for more information. -> env = await require("@cocalc/backend/nats/env").getEnv(); a = require("@cocalc/nats/system"); s = new a.SystemKv(env); await s.init(); - -*/ - -import { GeneralKV } from "@cocalc/nats/sync/general-kv"; - -export class SystemKv extends GeneralKV { - constructor(env) { - super({ env, name: "system" }); - } -} diff --git a/src/packages/nats/tiered-storage/client.ts b/src/packages/nats/tiered-storage/client.ts deleted file mode 100644 index 3144246c54..0000000000 --- a/src/packages/nats/tiered-storage/client.ts +++ /dev/null @@ -1,156 +0,0 @@ -/* -Client for the tiered server. -*/ - -import type { Info, Command } from "./server"; -import { tieredStorageSubject } from "./server"; -import { getEnv, getLogger } from "@cocalc/nats/client"; -import { type Location } from "@cocalc/nats/types"; -import { waitUntilConnected } from "@cocalc/nats/util"; -import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; -import { delay } from "awaiting"; - -const logger = getLogger("tiered-storage:client"); - -const TIMEOUT = { - restore: 90 * 1000, - backup: 90 * 1000, - archive: 90 * 1000, - info: 15 * 1000, -}; - -// Server will never ever archive anything that was active -// in less than this time, no matter what. Usually, it's much longer. -// This is what clients get to assume to reduce load. -export const MIN_ARCHIVE_TIME = 6 * 60 * 1000 * 60; // 6 hours - -const readyUntilAtLeast: { [location: string]: number } = {}; - -function toTime(s): number { - if (s == null) { - return 0; - } - return new Date(s).valueOf(); -} - -// 0 = never active -function lastActive(info: Info): number { - return Math.max( - toTime(info.nats.stream?.state.last_ts), - toTime(info.nats.kv?.state.last_ts), - ); -} - -// 0 = never backed up -// function lastBackup(info: Info): number { -// if (info.backup.stream == null) { -// return toTime(info.backup.kv?.ts); -// } -// if (info.backup.kv == null) { -// return toTime(info.backup.stream?.ts); -// } -// return Math.min(toTime(info.backup.stream?.ts), toTime(info.backup.kv?.ts)); -// } - -function stringToLocation(s: string): Location | null { - if (s.startsWith("account-")) { - return { account_id: s.slice("account-".length) }; - } else if (s.startsWith("project-")) { - return { project_id: s.slice("project-".length) }; - } - return null; -} - -export const waitUntilReady = reuseInFlight( - async (location: Location | string | null): Promise => { - if(location == null) { - return; - } - if (typeof location == "string") { - location = stringToLocation(location); - if (location == null) { - return; - } - } - if (process.env.COCALC_TEST_MODE) { - // no tiered storage in test mode - return; - } - const key = tieredStorageSubject(location); - if (readyUntilAtLeast[key] >= Date.now()) { - // definitely available - return; - } - logger.debug("waitUntilReady", location); - let d = 1000; - while (true) { - await waitUntilConnected(); - const locationInfo = await info(location); - const active = lastActive(locationInfo); - if (locationInfo.nats.kv != null || locationInfo.nats.stream != null) { - // it's live -- only question is how long is it guaranteed - readyUntilAtLeast[key] = MIN_ARCHIVE_TIME + active; - return; - } - // it's NOT live or it never existed - if ( - locationInfo.backup.kv == null && - locationInfo.backup.stream == null - ) { - // never existed, so will get created in the future - readyUntilAtLeast[key] = MIN_ARCHIVE_TIME + Date.now(); - return; - } - try { - // we have to restore - await restore(location); - } catch (err) { - // it may just be that two clients tried to restore at the same time and - // one wins. - d = Math.min(30000, d * 1.25 + Math.random()); - logger.debug( - `waitUntilReady -- WARNING: problem restoring archived nats data -- will retry in ${d}ms -- ${err}`, - ); - await delay(d); - continue; - } - // success - readyUntilAtLeast[key] = MIN_ARCHIVE_TIME + Date.now(); - return; - } - }, -); - -export async function restore(location: Location): Promise { - logger.debug("restore", location); - return (await call("restore", location)) as Info; -} - -export async function archive(location: Location): Promise { - logger.debug("archive", location); - return (await call("archive", location)) as Info; -} - -export async function backup(location: Location): Promise { - logger.debug("backup", location); - return (await call("backup", location)) as Info; -} - -export async function info(location: Location): Promise { - logger.debug("info", location); - return (await call("info", location)) as Info; -} - -async function call(command: Command, location: Location) { - const subject = tieredStorageSubject(location); - const { nc, jc } = await getEnv(); - const resp = await nc.request(subject, jc.encode({ command }), { - timeout: TIMEOUT[command], - }); - const x = jc.decode(resp.data); - if (x?.error) { - throw Error(x.error); - } else { - return x; - } -} diff --git a/src/packages/nats/tiered-storage/server.ts b/src/packages/nats/tiered-storage/server.ts deleted file mode 100644 index 6dffd5685b..0000000000 --- a/src/packages/nats/tiered-storage/server.ts +++ /dev/null @@ -1,166 +0,0 @@ -/* -NATS service that provides tiered storage of data. - -This is pure javascript and sets the basic interface, -behavior and types for both client and server. - -See also @cocalc/server/nats/tiered-storage. -*/ - -import { getEnv, getLogger } from "@cocalc/nats/client"; -import { type Subscription } from "@nats-io/nats-core"; -import { isValidUUID } from "@cocalc/util/misc"; -import { type Location } from "@cocalc/nats/types"; -import { delay } from "awaiting"; -import { type StreamInfo } from "@nats-io/jetstream"; - -const logger = getLogger("tiered-storage:server"); - -export type State = "archived" | "restoring" | "ready"; - -export interface Info { - nats: { stream: null | StreamInfo; kv: null | StreamInfo }; - backup: { stream: null | StreamInfo; kv: null | StreamInfo }; - location: Location; -} - -export const SUBJECT = "tiered-storage"; - -export interface TieredStorage { - info: (location: Location) => Promise; - restore: (location: Location) => Promise; - archive: (location: Location) => Promise; - backup: (location: Location) => Promise; - - // shut it down - close: () => Promise; -} - -export type Command = "restore" | "archive" | "backup" | "info"; - -export function tieredStorageSubject({ account_id, project_id }: Location) { - if (account_id) { - if (project_id) { - throw Error( - "location for tiered storage must specify exactly one of account_id or project_id, but it specifies both", - ); - } - if (!isValidUUID(account_id)) { - throw Error("invalid account_id"); - } - return `${SUBJECT}.account-${account_id}.api`; - } else if (project_id) { - if (!isValidUUID(project_id)) { - throw Error("invalid project_id"); - } - return `${SUBJECT}.project-${project_id}.api`; - } else { - throw Error( - "location for tiered storage must specify exactly one of account_id or project_id, but it specifies neither", - ); - } -} - -function getLocation(subject: string): Location { - if (subject.startsWith(`${SUBJECT}.account-`)) { - return { - account_id: subject.slice( - `${SUBJECT}.account-`.length, - `${SUBJECT}.account-`.length + 36, - ), - }; - } - if (subject.startsWith(`${SUBJECT}.project-`)) { - return { - project_id: subject.slice( - `${SUBJECT}.project-`.length, - `${SUBJECT}.project-`.length + 36, - ), - }; - } - throw Error(`invalid subject -- ${subject}`); -} - -let tieredStorage: TieredStorage | null = null; -export function init(ts: TieredStorage) { - logger.debug("init"); - if (tieredStorage != null) { - throw Error("tiered-storage: init already called"); - } - tieredStorage = ts; - mainLoop(); -} - -let terminated = false; -export async function terminate() { - logger.debug("terminate"); - if (terminated) { - return; - } - terminated = true; - if (tieredStorage) { - tieredStorage.close(); - } - tieredStorage = null; -} - -async function mainLoop() { - while (!terminated) { - logger.debug("mainLoop: running..."); - try { - await run(); - } catch (err) { - const DELAY = 5000; - logger.debug(`WARNING: run error (will restart in ${DELAY}ms) -- ${err}`); - await delay(DELAY); - } - } -} - -let sub: Subscription | null = null; -export async function run() { - const { nc } = await getEnv(); - const subject = `${SUBJECT}.*.api`; - logger.debug(`run: listening on '${subject}'`); - sub = nc.subscribe(subject, { queue: "0" }); - await listen(sub); -} - -async function listen(sub) { - logger.debug("listen"); - for await (const mesg of sub) { - if (tieredStorage == null) { - throw Error("tiered storage not available"); - } - handleMessage(mesg); - } -} - -async function handleMessage(mesg) { - let resp; - const { jc } = await getEnv(); - const location = getLocation(mesg.subject); - const { command } = jc.decode(mesg.data); - - try { - if (tieredStorage == null) { - throw Error("tiered storage not available"); - } - logger.debug("handleMessage", { location, command }); - if (command == "restore") { - resp = await tieredStorage.restore(location); - } else if (command == "archive") { - resp = await tieredStorage.archive(location); - } else if (command == "backup") { - resp = await tieredStorage.backup(location); - } else if (command == "info") { - resp = await tieredStorage.info(location); - } else { - throw Error(`unknown command '${command}'`); - } - } catch (err) { - resp = { error: `${err}` }; - } - //logger.debug("handleMessage -- resp", { location, command, resp }); - mesg.respond(jc.encode(resp)); -} diff --git a/src/packages/nats/types.ts b/src/packages/nats/types.ts deleted file mode 100644 index ba0d00b915..0000000000 --- a/src/packages/nats/types.ts +++ /dev/null @@ -1,30 +0,0 @@ -import type { NatsConnection as NatsConnection0 } from "@nats-io/nats-core"; -import type { EventEmitter } from "events"; -export type ValueType = "json" | "binary"; - -export type NatsConnection = NatsConnection0 & - Partial & { - getProjectPermissions?: () => Promise; - getConnectionInfo?: Function; - addProjectPermissions: (project_ids: string[]) => Promise; - }; - -export interface NatsEnv { - // nats connection, but frontend extends it to be an EventEmitter - nc: NatsConnection; - jc; // jsoncodec -} - -export type State = "disconnected" | "connected" | "closed"; - -export type NatsEnvFunction = () => Promise; - -export interface Location { - project_id?: string; - compute_server_id?: number; - - account_id?: string; - browser_id?: string; - - path?: string; -} diff --git a/src/packages/nats/util.ts b/src/packages/nats/util.ts deleted file mode 100644 index 5c821271a6..0000000000 --- a/src/packages/nats/util.ts +++ /dev/null @@ -1,167 +0,0 @@ -import jsonStableStringify from "json-stable-stringify"; -import type { MsgHdrs } from "@nats-io/nats-core"; -import { is_array } from "@cocalc/util/misc"; -import { encode as encodeBase64, decode as decodeBase64 } from "js-base64"; -export { encodeBase64, decodeBase64 }; -import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; -import { getConnection, getConnectionSync } from "./client"; -import { delay } from "awaiting"; - -// Get the number of NON-deleted keys in a nats kv store, matching a given subject: -// export async function numKeys(kv, x: string | string[] = ">"): Promise { -// let num = 0; -// for await (const _ of await kv.keys(x)) { -// num += 1; -// } -// return num; -// } - -// get everything from a KV store matching a subject pattern. -export async function getAllFromKv({ - kv, - key = ">", -}: { - kv; - key?: string | string[]; -}): Promise<{ - all: { [key: string]: any }; - revisions: { [key: string]: number }; - times: { [key: string]: Date }; - headers: { [key: string]: MsgHdrs }; -}> { - // const t = Date.now(); - // console.log("start getAllFromKv", key); - let all: any = {}; - let revisions: { [key: string]: number } = {}; - let times: { [key: string]: Date } = {}; - let headers: { [key: string]: MsgHdrs } = {}; - - if (is_array(key) && key.length > 1) { - // do all separately and combine... otherwise it hangs. - for (const k of key) { - const x = await getAllFromKv({ kv, key: k }); - all = { ...all, ...x.all }; - revisions = { ...revisions, ...x.revisions }; - times = { ...times, ...x.times }; - headers = { ...headers, ...x.headers }; - } - return { all, revisions, times, headers }; - } - - const watch = await kv.watch({ key, ignoreDeletes: false }); - if (watch._data._info.num_pending > 0) { - for await (const { key: key0, value, revision, sm } of watch) { - if (value.length > 0) { - // we MUST check value.length because we do NOT ignoreDeletes. - // we do NOT ignore deletes so that sm.di.pending goes down to 0. - // Otherwise, there is no way in general to know when we are done. - all[key0] = value; - revisions[key0] = revision; - times[key0] = sm.time; - headers[key0] = sm.headers; - } - if (sm.di.pending <= 0) { - // **NOTE! This will hang and never get hit if you don't have the $JC.FC.... auth enabled!!!!** - break; - } - } - } - watch.stop(); - // console.log("finished getAllFromKv", key, (Date.now() - t) / 1000, "seconds"); - return { all, revisions, times, headers }; -} - -export function handleErrorMessage(mesg) { - if (mesg?.error) { - if (mesg.error.startsWith("Error: ")) { - throw Error(mesg.error.slice("Error: ".length)); - } else { - throw Error(mesg.error); - } - } - return mesg; -} - -// Returns true if the subject matches the NATS pattern. -export function matchesPattern({ - pattern, - subject, -}: { - pattern: string; - subject: string; -}): boolean { - const subParts = subject.split("."); - const patParts = pattern.split("."); - let i = 0, - j = 0; - while (i < subParts.length && j < patParts.length) { - if (patParts[j] === ">") return true; - if (patParts[j] !== "*" && patParts[j] !== subParts[i]) return false; - i++; - j++; - } - - return i === subParts.length && j === patParts.length; -} - -// Converts the specified millis into Nanos -export type Nanos = number; -export function nanos(millis: number): Nanos { - return millis * 1000000; -} - -// Convert the specified Nanos into millis -export function millis(ns: Nanos): number { - return Math.floor(ns / 1000000); -} - -export function toKey(x): string | undefined { - if (x === undefined) { - return undefined; - } else if (typeof x === "object") { - return jsonStableStringify(x); - } else { - return `${x}`; - } -} - -// returns false if not connected or there is no connection yet. -export function isConnectedSync(): boolean { - const nc = getConnectionSync(); - // @ts-ignore - return !!nc?.protocol?.connected; -} - -export async function isConnected(nc?): Promise { - nc = nc ?? (await getConnection()); - // At least if this changes, things will be so broken, we'll quickly notice, hopefully. - // @ts-ignore - return !!nc.protocol?.connected; -} - -// Returns the max payload size for messages for the NATS server -// that we are connected to. This is used for chunking by the kv -// and stream to support arbitrarily large values. -export const getMaxPayload = reuseInFlight(async () => { - const nc = await getConnection(); - while (true) { - if (nc.info == null) { - await waitUntilConnected(); - await delay(100); - } else { - return nc.info.max_payload; - } - } -}); - -export const waitUntilConnected = reuseInFlight(async () => { - const nc = (await getConnection()) as any; - if (nc.protocol?.connected) { - return; - } - console.log("NATS waitUntilConnected: waiting..."); - while (!nc.protocol?.connected) { - await delay(500); - } - console.log("NATS waitUntilConnected: connected"); -}); diff --git a/src/packages/next/components/store/quota-config.tsx b/src/packages/next/components/store/quota-config.tsx index 81cb0423cc..c022a41f23 100644 --- a/src/packages/next/components/store/quota-config.tsx +++ b/src/packages/next/components/store/quota-config.tsx @@ -17,7 +17,6 @@ import { Typography, } from "antd"; import { useEffect, useRef, useState } from "react"; - import { Icon } from "@cocalc/frontend/components/icon"; import { displaySiteLicense } from "@cocalc/util/consts/site-license"; import { plural } from "@cocalc/util/misc"; @@ -151,14 +150,8 @@ export const QuotaConfig: React.FC = (props: Props) => { You selected a RAM quota of {ramVal}G. If your use-case involves a lot of RAM, consider using a{" "} - compute server - {" "} - or{" "} - - dedicated virtual machines + compute server. - . This will not only give you much more RAM, but also a far - superior experience! } /> @@ -199,7 +192,7 @@ export const QuotaConfig: React.FC = (props: Props) => { onChange(); }} units={"GB RAM"} - presets={boost ? [0, 2, 4, 8, 10] : [1, 2, 4, 8, 16]} + presets={boost ? [0, 2, 4, 8, 10] : [4, 8, 16]} /> ); @@ -557,9 +550,9 @@ export const QuotaConfig: React.FC = (props: Props) => { Configure the quotas you want to add on top of your existing - license. E.g. if your license provides a limit of 2 GB of RAM and - you add a matching boost license with 3 GB of RAM, you'll end up - with a total quota limit of 5 GB of RAM. + license. E.g. if your license provides a limit of 2 GB of RAM + and you add a matching boost license with 3 GB of RAM, you'll + end up with a total quota limit of 5 GB of RAM. diff --git a/src/packages/next/package.json b/src/packages/next/package.json index 34d5ac60c2..c761d4fe8f 100644 --- a/src/packages/next/package.json +++ b/src/packages/next/package.json @@ -27,6 +27,7 @@ "preinstall": "npx only-allow pnpm", "build": "pnpm patch-openapi && pnpm exec next telemetry disable && rm -rf .next && pnpm build-dev && NODE_OPTIONS='--max_old_space_size=8000' next build && pnpm generate-openapi", "build-dev": "pnpm patch-openapi && pnpm exec next telemetry disable && rm -rf dist && pnpm software && ../node_modules/.bin/tsc --build tsconfig-dist.json && pnpm generate-openapi", + "ts-build": "../node_modules/.bin/tsc --build tsconfig-dist.json", "tsc": "../node_modules/.bin/tsc --build tsconfig-dist.json -w --pretty --preserveWatchOutput ", "software": "bash ./software-inventory/setup.sh", "build-deps": "cd ../backend && pnpm build && cd ../util && pnpm build", @@ -35,8 +36,9 @@ "start": "unset PGHOST; pnpm exec next start", "start-project": "unset PGHOST PGUSER COCALC_ROOT; export PORT=5000 BASE_PATH=/$COCALC_PROJECT_ID/port/5000; echo https://cocalc.com$BASE_PATH; pnpm start", "test": "NODE_ENV='dev' pnpm exec jest && NODE_ENV='production' pnpm exec jest ./lib/api/framework.test.ts", + "depcheck": "pnpx depcheck --ignores @openapitools/openapi-generator-cli,eslint-config-next,locales,components,lib,public,pages,software-inventory,pg", "prepublishOnly": "pnpm test", - "patch-openapi": "sed -i '/^interface NrfOasData {/s/^interface/export interface/' node_modules/next-rest-framework/dist/index.d.ts", + "patch-openapi": "sed '/^interface NrfOasData {/s/^interface/export interface/' node_modules/next-rest-framework/dist/index.d.ts > node_modules/next-rest-framework/dist/index.d.ts.temp && mv node_modules/next-rest-framework/dist/index.d.ts.temp node_modules/next-rest-framework/dist/index.d.ts", "generate-openapi": "cd dist && rm -f public && ln -sf ../public public && NODE_ENV='dev' NODE_PATH=`pwd` npx next-rest-framework generate", "validate-openapi": "cd dist && rm -f public && ln -sf ../public public && NODE_ENV='dev' NODE_PATH=`pwd` npx next-rest-framework validate", "i18n:upload": "bash ./locales/upload.sh", @@ -55,7 +57,6 @@ "private": false, "dependencies": { "@ant-design/icons": "^6.0.0", - "@cocalc/assets": "workspace:*", "@cocalc/backend": "workspace:*", "@cocalc/cdn": "workspace:*", "@cocalc/database": "workspace:*", @@ -64,14 +65,12 @@ "@cocalc/util": "workspace:*", "@openapitools/openapi-generator-cli": "^2.19.1", "@types/react": "^18.3.10", - "@types/react-dom": "^18.3.0", "@vscode/vscode-languagedetection": "^1.0.22", "antd": "^5.24.7", "antd-img-crop": "^4.21.0", "awaiting": "^3.0.0", "base-64": "^1.0.0", "basic-auth": "^2.0.1", - "cookies": "^0.8.0", "csv-stringify": "^6.3.0", "dayjs": "^1.11.11", "express": "^4.21.2", @@ -86,22 +85,20 @@ "pg": "^8.7.1", "react": "^18.3.1", "react-dom": "^18.3.1", - "react-google-recaptcha": "^2.1.0", "react-google-recaptcha-v3": "^1.9.7", "react-intl": "^7.1.11", "serve-index": "^1.9.1", - "sharp": "^0.32.6", "timeago-react": "^3.0.4", - "tslib": "^2.3.1", "use-async-effect": "^2.2.7", "uuid": "^8.3.2", "xmlbuilder2": "^3.0.2", "zod": "^3.23.5" }, "devDependencies": { - "@babel/preset-typescript": "^7.23.3", + "@ant-design/cssinjs": "^1.23.0", "@types/express": "^4.17.21", "@types/node": "^18.16.14", + "@uiw/react-textarea-code-editor": "^3.1.1", "node-mocks-http": "^1.14.1", "react-test-renderer": "^18.2.0" } diff --git a/src/packages/next/pages/pricing/onprem.tsx b/src/packages/next/pages/pricing/onprem.tsx index c8e34a40a0..020ccf43af 100644 --- a/src/packages/next/pages/pricing/onprem.tsx +++ b/src/packages/next/pages/pricing/onprem.tsx @@ -338,13 +338,8 @@ function Body() { PostgreSQL {" "} - database and{" "} - - NATS.io - {" "} - communication service. PostgreSQL is used for persistent data - storage, and NATS for internal communication between CoCalc - services. + database. PostgreSQL is used for persistent data storage, and + Socket.io for internal communication between CoCalc services.
  • A shared network file-system like NFS. It must diff --git a/src/packages/next/tsconfig.json b/src/packages/next/tsconfig.json index d863adeded..da863e598f 100644 --- a/src/packages/next/tsconfig.json +++ b/src/packages/next/tsconfig.json @@ -38,7 +38,7 @@ { "path": "../backend" }, { "path": "../database" }, { "path": "../frontend" }, - { "path": "../nats" }, + { "path": "../conat" }, { "path": "../server" }, { "path": "../util" } ] diff --git a/src/packages/package.json b/src/packages/package.json index e52157eb4e..b3ef6c6886 100644 --- a/src/packages/package.json +++ b/src/packages/package.json @@ -30,10 +30,10 @@ "tar-fs@3.0.8": "3.0.9" }, "onlyBuiltDependencies": [ - "http-proxy-3", + "better-sqlite3", "websocket-sftp", "websocketfs", - "zeromq" + "zstd-napi" ] } } diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index 8ab9827741..e0272ea7ec 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -49,6 +49,9 @@ importers: '@cocalc/backend': specifier: workspace:* version: link:../backend + '@cocalc/util': + specifier: workspace:* + version: link:../util devDependencies: '@types/node': specifier: ^18.16.14 @@ -75,33 +78,30 @@ importers: '@cocalc/backend': specifier: workspace:* version: 'link:' - '@cocalc/nats': + '@cocalc/conat': specifier: workspace:* - version: link:../nats + version: link:../conat '@cocalc/util': specifier: workspace:* version: link:../util - '@nats-io/nkeys': - specifier: ^2.0.3 - version: 2.0.3 '@types/debug': specifier: ^4.1.12 version: 4.1.12 - '@types/watchpack': - specifier: ^2.4.4 - version: 2.4.4 - '@types/ws': - specifier: ^8.18.1 - version: 8.18.1 + '@types/jest': + specifier: ^29.5.14 + version: 29.5.14 awaiting: specifier: ^3.0.0 version: 3.0.0 + better-sqlite3: + specifier: ^11.10.0 + version: 11.10.0 chokidar: specifier: ^3.6.0 version: 3.6.0 debug: specifier: ^4.4.0 - version: 4.4.1(supports-color@9.4.0) + version: 4.4.1 fs-extra: specifier: ^11.2.0 version: 11.3.0 @@ -111,12 +111,6 @@ importers: lru-cache: specifier: ^7.18.3 version: 7.18.3 - nats: - specifier: ^2.29.3 - version: 2.29.3 - nats.ws: - specifier: ^1.30.2 - version: 1.30.3 password-hash: specifier: ^1.2.2 version: 1.2.2 @@ -129,25 +123,16 @@ importers: shell-escape: specifier: ^0.2.0 version: 0.2.0 - supports-color: - specifier: ^9.0.2 - version: 9.4.0 tmp-promise: specifier: ^3.0.3 version: 3.0.3 - underscore: - specifier: ^1.12.1 - version: 1.13.7 - ws: - specifier: ^8.18.0 - version: 8.18.2 + zstd-napi: + specifier: ^0.0.10 + version: 0.0.10 devDependencies: '@types/node': specifier: ^18.16.14 version: 18.19.111 - expect: - specifier: ^26.6.2 - version: 26.6.2 cdn: devDependencies: @@ -163,9 +148,6 @@ importers: '@cocalc/comm': specifier: workspace:* version: 'link:' - '@cocalc/jupyter': - specifier: workspace:* - version: link:../jupyter '@cocalc/sync': specifier: workspace:* version: link:../sync @@ -177,23 +159,93 @@ importers: specifier: ^18.16.14 version: 18.19.111 + conat: + dependencies: + '@cocalc/comm': + specifier: workspace:* + version: link:../comm + '@cocalc/conat': + specifier: workspace:* + version: 'link:' + '@cocalc/redis-streams-adapter': + specifier: ^0.2.3 + version: 0.2.3(socket.io-adapter@2.5.5) + '@cocalc/util': + specifier: workspace:* + version: link:../util + '@isaacs/ttlcache': + specifier: ^1.4.1 + version: 1.4.1 + '@msgpack/msgpack': + specifier: ^3.1.1 + version: 3.1.2 + '@socket.io/redis-adapter': + specifier: ^8.3.0 + version: 8.3.0(socket.io-adapter@2.5.5) + ascii-table3: + specifier: ^1.0.1 + version: 1.0.1 + awaiting: + specifier: ^3.0.0 + version: 3.0.0 + consistent-hash: + specifier: ^1.2.2 + version: 1.2.2 + dayjs: + specifier: ^1.11.11 + version: 1.11.13 + events: + specifier: 3.3.0 + version: 3.3.0 + immutable: + specifier: ^4.3.0 + version: 4.3.7 + iovalkey: + specifier: ^0.3.1 + version: 0.3.3 + js-base64: + specifier: ^3.7.7 + version: 3.7.7 + json-stable-stringify: + specifier: ^1.0.1 + version: 1.3.0 + lodash: + specifier: ^4.17.21 + version: 4.17.21 + socket.io: + specifier: ^4.8.1 + version: 4.8.1 + socket.io-client: + specifier: ^4.8.1 + version: 4.8.1 + devDependencies: + '@types/better-sqlite3': + specifier: ^7.6.13 + version: 7.6.13 + '@types/json-stable-stringify': + specifier: ^1.0.32 + version: 1.2.0 + '@types/lodash': + specifier: ^4.14.202 + version: 4.17.17 + '@types/node': + specifier: ^18.16.14 + version: 18.19.111 + database: dependencies: '@cocalc/backend': specifier: workspace:* version: link:../backend + '@cocalc/conat': + specifier: workspace:* + version: link:../conat '@cocalc/database': specifier: workspace:* version: 'link:' - '@cocalc/nats': - specifier: workspace:* - version: link:../nats '@cocalc/util': specifier: workspace:* version: link:../util - '@nats-io/services': - specifier: 3.0.0 - version: 3.0.0 async: specifier: ^1.5.2 version: 1.5.2 @@ -202,25 +254,16 @@ importers: version: 3.0.0 debug: specifier: ^4.4.0 - version: 4.4.1(supports-color@8.1.1) + version: 4.4.1 immutable: specifier: ^4.3.0 version: 4.3.7 - json-stable-stringify: - specifier: ^1.0.1 - version: 1.3.0 lodash: specifier: ^4.17.21 version: 4.17.21 lru-cache: specifier: ^7.18.3 version: 7.18.3 - nats: - specifier: ^2.29.3 - version: 2.29.3 - node-fetch: - specifier: 2.6.7 - version: 2.6.7(encoding@0.1.13) pg: specifier: ^8.7.1 version: 8.16.0 @@ -233,9 +276,6 @@ importers: sql-string-escape: specifier: ^1.1.6 version: 1.1.6 - uuid: - specifier: ^8.3.2 - version: 8.3.2 validator: specifier: ^13.6.0 version: 13.15.15 @@ -249,9 +289,6 @@ importers: '@types/pg': specifier: ^8.6.1 version: 8.15.4 - '@types/uuid': - specifier: ^8.3.1 - version: 8.3.4 coffeescript: specifier: ^2.5.1 version: 2.7.0 @@ -261,12 +298,12 @@ importers: '@cocalc/backend': specifier: workspace:* version: link:../backend + '@cocalc/conat': + specifier: workspace:* + version: link:../conat '@cocalc/file-server': specifier: workspace:* version: 'link:' - '@cocalc/nats': - specifier: workspace:* - version: link:../nats '@cocalc/util': specifier: workspace:* version: link:../util @@ -274,14 +311,14 @@ importers: specifier: ^3.0.0 version: 3.0.0 better-sqlite3: - specifier: ^11.8.1 + specifier: ^11.10.0 version: 11.10.0 lodash: specifier: ^4.17.21 version: 4.17.21 devDependencies: '@types/better-sqlite3': - specifier: ^7.6.12 + specifier: ^7.6.13 version: 7.6.13 '@types/lodash': specifier: ^4.14.202 @@ -307,6 +344,9 @@ importers: '@cocalc/comm': specifier: workspace:* version: link:../comm + '@cocalc/conat': + specifier: workspace:* + version: link:../conat '@cocalc/frontend': specifier: workspace:* version: 'link:' @@ -316,9 +356,6 @@ importers: '@cocalc/local-storage-lru': specifier: ^2.4.3 version: 2.5.0 - '@cocalc/nats': - specifier: workspace:* - version: link:../nats '@cocalc/sync': specifier: workspace:* version: link:../sync @@ -341,7 +378,7 @@ importers: specifier: ^3.2.1 version: 3.2.2(react@18.3.1) '@isaacs/ttlcache': - specifier: ^1.2.1 + specifier: ^1.4.1 version: 1.4.1 '@jupyter-widgets/base': specifier: ^4.1.1 @@ -352,24 +389,9 @@ importers: '@jupyter-widgets/output': specifier: ^4.1.0 version: 4.1.7(crypto@1.0.1)(react@18.3.1) - '@lumino/widgets': - specifier: ^1.31.1 - version: 1.37.2(crypto@1.0.1) '@microlink/react-json-view': specifier: ^1.23.3 version: 1.26.2(@types/react@18.3.23)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) - '@nats-io/jetstream': - specifier: 3.0.0 - version: 3.0.0 - '@nats-io/kv': - specifier: 3.0.0 - version: 3.0.0 - '@nats-io/nats-core': - specifier: 3.0.0 - version: 3.0.0 - '@nats-io/services': - specifier: 3.0.0 - version: 3.0.0 '@orama/orama': specifier: 3.0.0-rc-3 version: 3.0.0-rc-3 @@ -379,9 +401,6 @@ importers: '@rinsuki/lz4-ts': specifier: ^1.0.1 version: 1.0.1 - '@speed-highlight/core': - specifier: ^1.1.11 - version: 1.2.7 '@stripe/react-stripe-js': specifier: ^3.1.1 version: 3.7.0(@stripe/stripe-js@5.10.0)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) @@ -392,11 +411,23 @@ importers: specifier: ^4.1.12 version: 4.1.12 '@uiw/react-textarea-code-editor': - specifier: ^2.1.1 - version: 2.1.9(@babel/runtime@7.27.6)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + specifier: ^3.1.1 + version: 3.1.1(@babel/runtime@7.27.6)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) '@use-gesture/react': specifier: ^10.2.24 version: 10.3.1(react@18.3.1) + '@xterm/addon-fit': + specifier: ^0.10.0 + version: 0.10.0(@xterm/xterm@5.5.0) + '@xterm/addon-web-links': + specifier: ^0.11.0 + version: 0.11.0(@xterm/xterm@5.5.0) + '@xterm/addon-webgl': + specifier: ^0.18.0 + version: 0.18.0(@xterm/xterm@5.5.0) + '@xterm/xterm': + specifier: ^5.5.0 + version: 5.5.0 '@zippytech/react-notify-resize': specifier: ^4.0.4 version: 4.0.4(prop-types@15.8.1)(react@18.3.1) @@ -418,15 +449,6 @@ importers: awaiting: specifier: ^3.0.0 version: 3.0.0 - bootbox: - specifier: ^4.4.0 - version: 4.4.0 - bootstrap: - specifier: '=3.4.1' - version: 3.4.1 - bootstrap-colorpicker: - specifier: ^2.5.3 - version: 2.5.3 cat-names: specifier: ^3.1.0 version: 3.1.0 @@ -445,9 +467,6 @@ importers: create-react-class: specifier: ^15.7.0 version: 15.7.0 - css-color-names: - specifier: 0.0.4 - version: 0.0.4 csv-parse: specifier: ^5.3.6 version: 5.6.0 @@ -465,7 +484,7 @@ importers: version: 1.11.13 debug: specifier: ^4.4.0 - version: 4.4.1(supports-color@8.1.1) + version: 4.4.1 direction: specifier: ^1.0.4 version: 1.0.4 @@ -487,24 +506,12 @@ importers: events: specifier: 3.3.0 version: 3.3.0 - expect: - specifier: ^26.6.2 - version: 26.6.2 - fflate: - specifier: 0.7.3 - version: 0.7.3 gpt3-tokenizer: specifier: ^1.1.5 version: 1.1.5 - history: - specifier: ^1.17.0 - version: 1.17.0 html-react-parser: specifier: ^1.4.14 version: 1.4.14(react@18.3.1) - htmlparser: - specifier: ^1.7.7 - version: 1.7.7 humanize-list: specifier: ^1.0.1 version: 1.0.1 @@ -514,30 +521,12 @@ importers: immutable: specifier: ^4.3.0 version: 4.3.7 - install: - specifier: ^0.13.0 - version: 0.13.0 is-hotkey: specifier: ^0.2.0 version: 0.2.0 jquery: specifier: ^3.6.0 version: 3.7.1 - jquery-tooltip: - specifier: ^0.2.1 - version: 0.2.1(jquery-focus-flyout@0.0.5(jquery-focus-exit@1.0.1(jquery@3.7.1))(jquery-next-id@1.0.1(jquery@3.7.1))(jquery@3.7.1))(jquery-hover-flyout@0.0.5(jquery-mouse-exit@1.0.1(jquery@3.7.1))(jquery-next-id@1.0.1(jquery@3.7.1))(jquery@3.7.1))(jquery@3.7.1) - jquery-ui: - specifier: ^1.14.0 - version: 1.14.1 - jquery-ui-touch-punch: - specifier: ^0.2 - version: 0.2.3 - jquery.payment: - specifier: ^3.0.0 - version: 3.0.0 - jquery.scrollintoview: - specifier: ^1.9 - version: 1.9.4 js-cookie: specifier: ^2.2.1 version: 2.2.1 @@ -580,27 +569,15 @@ importers: mermaid: specifier: ^11.4.1 version: 11.6.0 - nats.ws: - specifier: ^1.30.2 - version: 1.30.3 node-forge: specifier: ^1.0.0 version: 1.3.1 - octicons: - specifier: ^3.5.0 - version: 3.5.0 onecolor: specifier: ^3.1.0 version: 3.1.0 pdfjs-dist: specifier: ^4.6.82 version: 4.10.38 - pegjs: - specifier: ^0.10.0 - version: 0.10.0 - pica: - specifier: ^7.1.0 - version: 7.1.1 plotly.js: specifier: ^2.29.1 version: 2.35.3(@rspack/core@1.3.15(@swc/helpers@0.5.5))(mapbox-gl@1.13.3)(webpack@5.99.5) @@ -610,9 +587,6 @@ importers: prop-types: specifier: ^15.7.2 version: 15.8.1 - punycode: - specifier: 2.3.1 - version: 2.3.1 re-resizable: specifier: ^6.9.0 version: 6.11.2(react-dom@18.3.1(react@18.3.1))(react@18.3.1) @@ -658,12 +632,6 @@ importers: react-virtuoso: specifier: ^4.9.0 version: 4.12.8(react-dom@18.3.1(react@18.3.1))(react@18.3.1) - shallowequal: - specifier: ^1.1.0 - version: 1.1.0 - shell-escape: - specifier: ^0.2.0 - version: 0.2.0 slate: specifier: ^0.103.0 version: 0.103.0 @@ -673,12 +641,6 @@ importers: three-ancient: specifier: npm:three@=0.78.0 version: three@0.78.0 - timeago: - specifier: ^1.6.3 - version: 1.6.7 - tslib: - specifier: ^2.3.1 - version: 2.8.1 underscore: specifier: ^1.12.1 version: 1.13.7 @@ -706,27 +668,18 @@ importers: xss: specifier: ^1.0.11 version: 1.0.15 - xterm: - specifier: 5.0.0 - version: 5.0.0 - xterm-addon-fit: - specifier: ^0.6.0 - version: 0.6.0(xterm@5.0.0) - xterm-addon-web-links: - specifier: ^0.7.0 - version: 0.7.0(xterm@5.0.0) - xterm-addon-webgl: - specifier: ^0.13.0 - version: 0.13.0(xterm@5.0.0) zlibjs: specifier: ^0.3.1 version: 0.3.1 devDependencies: + '@cfaester/enzyme-adapter-react-18': + specifier: ^0.8.0 + version: 0.8.0(enzyme@3.11.0)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) '@cspell/dict-typescript': specifier: ^3.2.0 version: 3.2.2 '@formatjs/cli': - specifier: ^6.2.12 + specifier: ^6.7.1 version: 6.7.1 '@types/codemirror': specifier: ^5.60.15 @@ -746,12 +699,6 @@ importers: '@types/md5': specifier: ^2.2.0 version: 2.3.5 - '@types/mocha': - specifier: ^10.0.0 - version: 10.0.10 - '@types/pica': - specifier: ^5.1.3 - version: 5.1.3 '@types/react': specifier: ^18.3.10 version: 18.3.23 @@ -767,12 +714,15 @@ importers: cspell: specifier: ^8.17.2 version: 8.19.4 - mocha: - specifier: ^10.0.0 - version: 10.8.2 + enzyme: + specifier: ^3.11.0 + version: 3.11.0 react-test-renderer: specifier: ^18.2.0 version: 18.3.1(react@18.3.1) + tsd: + specifier: ^0.22.0 + version: 0.22.0 type-fest: specifier: ^3.3.0 version: 3.13.1 @@ -788,18 +738,15 @@ importers: '@cocalc/cdn': specifier: workspace:* version: link:../cdn + '@cocalc/conat': + specifier: workspace:* + version: link:../conat '@cocalc/database': specifier: workspace:* version: link:../database - '@cocalc/frontend': - specifier: workspace:* - version: link:../frontend '@cocalc/hub': specifier: workspace:* version: 'link:' - '@cocalc/nats': - specifier: workspace:* - version: link:../nats '@cocalc/next': specifier: workspace:* version: link:../next @@ -813,38 +760,17 @@ importers: specifier: workspace:* version: link:../util '@isaacs/ttlcache': - specifier: ^1.2.1 + specifier: ^1.4.1 version: 1.4.1 - '@passport-next/passport-google-oauth2': - specifier: ^1.0.0 - version: 1.0.0 - '@passport-next/passport-oauth2': - specifier: ^2.1.4 - version: 2.1.4 '@types/formidable': specifier: ^3.4.5 version: 3.4.5 - '@types/primus': - specifier: ^7.3.9 - version: 7.3.9 - '@types/react': - specifier: ^18.3.10 - version: 18.3.23 - '@types/uuid': - specifier: ^8.3.1 - version: 8.3.4 async: specifier: ^1.5.2 version: 1.5.2 awaiting: specifier: ^3.0.0 version: 3.0.0 - basic-auth: - specifier: ^2.0.1 - version: 2.0.1 - bindings: - specifier: ^1.3.0 - version: 1.5.0 blocked: specifier: ^1.1.0 version: 1.3.0 @@ -865,10 +791,7 @@ importers: version: 2.8.5 debug: specifier: ^4.4.0 - version: 4.4.1(supports-color@8.1.1) - escape-html: - specifier: ^1.0.3 - version: 1.0.3 + version: 4.4.1 express: specifier: ^4.21.2 version: 4.21.2 @@ -876,53 +799,23 @@ importers: specifier: ^3.5.4 version: 3.5.4 http-proxy-3: - specifier: ^1.20.0 + specifier: ^1.20.5 version: 1.20.5 - immutable: - specifier: ^4.3.0 - version: 4.3.7 - jquery: - specifier: ^3.6.0 - version: 3.7.1 - json-stable-stringify: - specifier: ^1.0.1 - version: 1.3.0 lodash: specifier: ^4.17.21 version: 4.17.21 lru-cache: specifier: ^7.18.3 version: 7.18.3 - mime: - specifier: ^1.3.4 - version: 1.6.0 mime-types: specifier: ^2.1.35 version: 2.1.35 - mkdirp: - specifier: ^1.0.4 - version: 1.0.4 ms: specifier: 2.1.2 version: 2.1.2 - nats: - specifier: ^2.29.3 - version: 2.29.3 - next: - specifier: 14.2.28 - version: 14.2.28(@babel/core@7.26.9)(@playwright/test@1.51.1)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.89.2) parse-domain: specifier: ^5.0.0 version: 5.0.0(encoding@0.1.13) - passport: - specifier: ^0.6.0 - version: 0.6.0 - password-hash: - specifier: ^1.2.2 - version: 1.2.2 - primus: - specifier: ^8.0.9 - version: 8.0.9 prom-client: specifier: ^13.0.0 version: 13.2.0 @@ -932,27 +825,6 @@ importers: react: specifier: ^18.3.1 version: 18.3.1 - react-dom: - specifier: ^18.3.1 - version: 18.3.1(react@18.3.1) - read: - specifier: ^1.0.7 - version: 1.0.7 - require-reload: - specifier: ^0.2.2 - version: 0.2.2 - safe-json-stringify: - specifier: ^1.2.0 - version: 1.2.0 - serve-index: - specifier: ^1.9.1 - version: 1.9.1 - sql-string-escape: - specifier: ^1.1.6 - version: 1.1.6 - temp: - specifier: ^0.9.4 - version: 0.9.4 uglify-js: specifier: ^3.14.1 version: 3.19.3 @@ -971,9 +843,6 @@ importers: webpack-hot-middleware: specifier: ^2.26.1 version: 2.26.1 - ws: - specifier: ^8.18.0 - version: 8.18.2 devDependencies: '@types/express': specifier: ^4.17.21 @@ -981,69 +850,36 @@ importers: '@types/node': specifier: ^18.16.14 version: 18.19.111 - '@types/passport': - specifier: ^1.0.9 - version: 1.0.17 - '@types/react-dom': - specifier: ^18.3.0 - version: 18.3.7(@types/react@18.3.23) coffeescript: specifier: ^2.5.1 version: 2.7.0 - expect: - specifier: ^26.6.2 - version: 26.6.2 - node-cjsx: - specifier: ^2.0.0 - version: 2.0.0 - should: - specifier: ^7.1.1 - version: 7.1.1 - sinon: - specifier: ^4.5.0 - version: 4.5.0 jupyter: dependencies: '@cocalc/backend': specifier: workspace:* version: link:../backend + '@cocalc/conat': + specifier: workspace:* + version: link:../conat '@cocalc/jupyter': specifier: workspace:* version: 'link:' - '@cocalc/nats': - specifier: workspace:* - version: link:../nats '@cocalc/sync': specifier: workspace:* version: link:../sync - '@cocalc/sync-client': - specifier: workspace:* - version: link:../sync-client '@cocalc/util': specifier: workspace:* version: link:../util '@nteract/messaging': specifier: ^7.0.20 version: 7.0.20 - '@types/json-stable-stringify': - specifier: ^1.0.32 - version: 1.2.0 - '@types/node-cleanup': - specifier: ^2.1.2 - version: 2.1.5 awaiting: specifier: ^3.0.0 version: 3.0.0 debug: specifier: ^4.4.0 - version: 4.4.1(supports-color@8.1.1) - enchannel-zmq-backend: - specifier: ^9.1.23 - version: 9.1.23(rxjs@7.8.2) - execa: - specifier: ^8.0.1 - version: 8.0.1 + version: 4.4.1 expect: specifier: ^26.6.2 version: 26.6.2 @@ -1077,83 +913,34 @@ importers: node-cleanup: specifier: ^2.1.2 version: 2.1.2 - portfinder: - specifier: ^1.0.32 - version: 1.0.37 + rxjs: + specifier: ^6.6.7 + version: 6.6.7 shell-escape: specifier: ^0.2.0 version: 0.2.0 - tsimportlib: - specifier: ^0.0.5 - version: 0.0.5 uuid: specifier: ^8.3.2 version: 8.3.2 - devDependencies: - '@types/node': - specifier: ^18.16.14 - version: 18.19.111 - - nats: - dependencies: - '@cocalc/comm': - specifier: workspace:* - version: link:../comm - '@cocalc/nats': - specifier: workspace:* - version: 'link:' - '@cocalc/util': - specifier: workspace:* - version: link:../util - '@nats-io/jetstream': - specifier: 3.0.0 - version: 3.0.0 - '@nats-io/kv': - specifier: 3.0.0 - version: 3.0.0 - '@nats-io/nats-core': - specifier: 3.0.0 - version: 3.0.0 - '@nats-io/services': - specifier: 3.0.0 - version: 3.0.0 - awaiting: - specifier: ^3.0.0 - version: 3.0.0 - events: - specifier: 3.3.0 - version: 3.3.0 - immutable: - specifier: ^4.3.0 - version: 4.3.7 - js-base64: - specifier: ^3.7.7 - version: 3.7.7 - json-stable-stringify: - specifier: ^1.0.1 - version: 1.3.0 - lodash: - specifier: ^4.17.21 - version: 4.17.21 + zeromq: + specifier: ^6.4.2 + version: 6.4.2 devDependencies: '@types/json-stable-stringify': specifier: ^1.0.32 version: 1.2.0 - '@types/lodash': - specifier: ^4.14.202 - version: 4.17.17 '@types/node': specifier: ^18.16.14 version: 18.19.111 + '@types/node-cleanup': + specifier: ^2.1.2 + version: 2.1.5 next: dependencies: '@ant-design/icons': specifier: ^6.0.0 version: 6.0.0(react-dom@18.3.1(react@18.3.1))(react@18.3.1) - '@cocalc/assets': - specifier: workspace:* - version: link:../assets '@cocalc/backend': specifier: workspace:* version: link:../backend @@ -1178,9 +965,6 @@ importers: '@types/react': specifier: ^18.3.10 version: 18.3.23 - '@types/react-dom': - specifier: ^18.3.0 - version: 18.3.7(@types/react@18.3.23) '@vscode/vscode-languagedetection': specifier: ^1.0.22 version: 1.0.22 @@ -1199,9 +983,6 @@ importers: basic-auth: specifier: ^2.0.1 version: 2.0.1 - cookies: - specifier: ^0.8.0 - version: 0.8.0 csv-stringify: specifier: ^6.3.0 version: 6.5.2 @@ -1244,9 +1025,6 @@ importers: react-dom: specifier: ^18.3.1 version: 18.3.1(react@18.3.1) - react-google-recaptcha: - specifier: ^2.1.0 - version: 2.1.0(react@18.3.1) react-google-recaptcha-v3: specifier: ^1.9.7 version: 1.11.0(react-dom@18.3.1(react@18.3.1))(react@18.3.1) @@ -1256,15 +1034,9 @@ importers: serve-index: specifier: ^1.9.1 version: 1.9.1 - sharp: - specifier: ^0.32.6 - version: 0.32.6 timeago-react: specifier: ^3.0.4 version: 3.0.7(react@18.3.1) - tslib: - specifier: ^2.3.1 - version: 2.8.1 use-async-effect: specifier: ^2.2.7 version: 2.2.7(react@18.3.1) @@ -1278,15 +1050,18 @@ importers: specifier: ^3.23.5 version: 3.25.64 devDependencies: - '@babel/preset-typescript': - specifier: ^7.23.3 - version: 7.27.1(@babel/core@7.26.9) + '@ant-design/cssinjs': + specifier: ^1.23.0 + version: 1.23.0(react-dom@18.3.1(react@18.3.1))(react@18.3.1) '@types/express': specifier: ^4.17.21 version: 4.17.23 '@types/node': specifier: ^18.16.14 version: 18.19.111 + '@uiw/react-textarea-code-editor': + specifier: ^3.1.1 + version: 3.1.1(@babel/runtime@7.27.6)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) node-mocks-http: specifier: ^1.14.1 version: 1.17.2(@types/express@4.17.23)(@types/node@18.19.111) @@ -1302,12 +1077,12 @@ importers: '@cocalc/comm': specifier: workspace:* version: link:../comm + '@cocalc/conat': + specifier: workspace:* + version: link:../conat '@cocalc/jupyter': specifier: workspace:* version: link:../jupyter - '@cocalc/nats': - specifier: workspace:* - version: link:../nats '@cocalc/primus-multiplex': specifier: ^1.1.0 version: 1.1.0 @@ -1326,27 +1101,15 @@ importers: '@cocalc/sync-fs': specifier: workspace:* version: link:../sync-fs - '@cocalc/terminal': - specifier: workspace:* - version: link:../terminal '@cocalc/util': specifier: workspace:* version: link:../util '@lydell/node-pty': specifier: ^1.1.0 version: 1.1.0 - '@nats-io/jetstream': - specifier: 3.0.0 - version: 3.0.0 - '@nats-io/kv': - specifier: 3.0.0 - version: 3.0.0 - '@nats-io/services': - specifier: 3.0.0 - version: 3.0.0 - '@nteract/messaging': - specifier: ^7.0.20 - version: 7.0.20 + '@types/jest': + specifier: ^29.5.14 + version: 29.5.14 awaiting: specifier: ^3.0.0 version: 3.0.0 @@ -1362,15 +1125,9 @@ importers: daemonize-process: specifier: ^3.0.0 version: 3.0.0 - debug: - specifier: ^4.4.0 - version: 4.4.1(supports-color@8.1.1) diskusage: specifier: ^1.1.3 version: 1.2.0 - expect: - specifier: ^26.6.2 - version: 26.6.2 express: specifier: ^4.21.2 version: 4.21.2 @@ -1380,30 +1137,9 @@ importers: get-port: specifier: ^5.1.1 version: 5.1.1 - googlediff: - specifier: ^0.1.0 - version: 0.1.0 - json-stable-stringify: - specifier: ^1.0.1 - version: 1.3.0 - jupyter-paths: - specifier: ^2.0.3 - version: 2.0.4 - lean-client-js-node: - specifier: ^1.2.12 - version: 1.5.0 lodash: specifier: ^4.17.21 version: 4.17.21 - lru-cache: - specifier: ^7.18.3 - version: 7.18.3 - nats: - specifier: ^2.29.3 - version: 2.29.3 - pidusage: - specifier: ^1.2.0 - version: 1.2.0 prettier: specifier: ^3.0.2 version: 3.5.3 @@ -1422,9 +1158,6 @@ importers: tmp: specifier: 0.0.33 version: 0.0.33 - uglify-js: - specifier: ^3.14.1 - version: 3.19.3 uuid: specifier: ^8.3.2 version: 8.3.2 @@ -1437,9 +1170,6 @@ importers: ws: specifier: ^8.18.0 version: 8.18.2 - zeromq: - specifier: ^5.2.8 - version: 5.3.1 devDependencies: '@types/body-parser': specifier: ^1.19.5 @@ -1447,9 +1177,6 @@ importers: '@types/express': specifier: ^4.17.21 version: 4.17.23 - '@types/jquery': - specifier: ^3.5.5 - version: 3.5.32 '@types/lodash': specifier: ^4.14.202 version: 4.17.17 @@ -1468,24 +1195,21 @@ importers: '@cocalc/backend': specifier: workspace:* version: link:../backend + '@cocalc/conat': + specifier: workspace:* + version: link:../conat '@cocalc/database': specifier: workspace:* version: link:../database '@cocalc/gcloud-pricing-calculator': specifier: ^1.17.0 version: 1.17.0 - '@cocalc/nats': - specifier: workspace:* - version: link:../nats '@cocalc/server': specifier: workspace:* version: 'link:' '@cocalc/util': specifier: workspace:* version: link:../util - '@google-ai/generativelanguage': - specifier: ^3.1.0 - version: 3.2.0 '@google-cloud/bigquery': specifier: ^7.8.0 version: 7.9.4(encoding@0.1.13) @@ -1505,7 +1229,7 @@ importers: specifier: ^0.14.0 version: 0.14.1 '@isaacs/ttlcache': - specifier: ^1.2.1 + specifier: ^1.4.1 version: 1.4.1 '@langchain/anthropic': specifier: ^0.3.18 @@ -1515,7 +1239,7 @@ importers: version: 0.3.58(openai@4.104.0(encoding@0.1.13)(ws@8.18.2)(zod@3.25.32)) '@langchain/google-genai': specifier: ^0.2.4 - version: 0.2.11(@langchain/core@0.3.58(openai@4.104.0(encoding@0.1.13)(ws@8.18.2)(zod@3.25.32))) + version: 0.2.12(@langchain/core@0.3.58(openai@4.104.0(encoding@0.1.13)(ws@8.18.2)(zod@3.25.32))) '@langchain/mistralai': specifier: ^0.2.0 version: 0.2.1(@langchain/core@0.3.58(openai@4.104.0(encoding@0.1.13)(ws@8.18.2)(zod@3.25.32)))(zod@3.25.32) @@ -1525,24 +1249,6 @@ importers: '@langchain/openai': specifier: ^0.5.5 version: 0.5.13(@langchain/core@0.3.58(openai@4.104.0(encoding@0.1.13)(ws@8.18.2)(zod@3.25.32)))(encoding@0.1.13)(ws@8.18.2) - '@nats-io/jetstream': - specifier: 3.0.0 - version: 3.0.0 - '@nats-io/jwt': - specifier: 0.0.10-5 - version: 0.0.10-5 - '@nats-io/kv': - specifier: 3.0.0 - version: 3.0.0 - '@nats-io/nats-core': - specifier: 3.0.0 - version: 3.0.0 - '@nats-io/nkeys': - specifier: ^2.0.3 - version: 2.0.3 - '@nats-io/services': - specifier: 3.0.0 - version: 3.0.0 '@node-saml/passport-saml': specifier: ^5.0.1 version: 5.0.1 @@ -1561,6 +1267,12 @@ importers: '@sendgrid/mail': specifier: ^8.1.4 version: 8.1.5 + '@socket.io/cluster-adapter': + specifier: ^0.2.2 + version: 0.2.2(socket.io-adapter@2.5.5) + '@socket.io/sticky': + specifier: ^1.0.4 + version: 1.0.4 '@zxcvbn-ts/core': specifier: ^3.0.4 version: 3.0.4 @@ -1581,19 +1293,19 @@ importers: version: 3.0.0 axios: specifier: ^1.7.5 - version: 1.9.0 + version: 1.10.0 base62: specifier: ^2.0.1 version: 2.0.2 base64-js: specifier: ^1.5.1 version: 1.5.1 - bottleneck: - specifier: ^2.19.5 - version: 2.19.5 cloudflare: specifier: ^2.9.1 version: 2.9.1 + cookie: + specifier: ^1.0.0 + version: 1.0.2 cookies: specifier: ^0.8.0 version: 0.8.0 @@ -1603,6 +1315,9 @@ importers: dot-object: specifier: ^2.1.5 version: 2.1.5 + express: + specifier: ^4.21.2 + version: 4.21.2 express-session: specifier: ^1.18.1 version: 1.18.1 @@ -1636,18 +1351,12 @@ importers: markdown-it: specifier: ^13.0.1 version: 13.0.2 - mkdirp: - specifier: ^1.0.4 - version: 1.0.4 ms: specifier: 2.1.2 version: 2.1.2 nanoid: specifier: ^3.3.8 version: 3.3.11 - nats: - specifier: ^2.29.3 - version: 2.29.3 node-zendesk: specifier: ^5.0.13 version: 5.0.15(encoding@0.1.13) @@ -1684,9 +1393,6 @@ importers: passport-google-oauth20: specifier: ^2.0.0 version: 2.0.0 - passport-ldapauth: - specifier: ^3.0.1 - version: 3.0.1 passport-oauth: specifier: ^1.0.0 version: 1.0.0 @@ -1727,6 +1433,9 @@ importers: '@types/express-session': specifier: ^1.18.0 version: 1.18.2 + '@types/jest': + specifier: ^29.5.14 + version: 29.5.14 '@types/lodash': specifier: ^4.14.202 version: 4.17.17 @@ -1774,13 +1483,13 @@ importers: version: link:../util devDependencies: '@rspack/cli': - specifier: ^1.1.1 + specifier: ^1.3.11 version: 1.3.15(@rspack/core@1.3.15(@swc/helpers@0.5.5))(@types/express@4.17.23)(webpack@5.99.5) '@rspack/core': - specifier: ^1.1.1 + specifier: ^1.3.11 version: 1.3.15(@swc/helpers@0.5.5) '@rspack/plugin-react-refresh': - specifier: ^1.0.0 + specifier: ^1.4.3 version: 1.4.3(react-refresh@0.14.2)(webpack-hot-middleware@2.26.1) '@types/jquery': specifier: ^3.5.5 @@ -1956,9 +1665,9 @@ importers: sync: dependencies: - '@cocalc/nats': + '@cocalc/conat': specifier: workspace:* - version: link:../nats + version: link:../conat '@cocalc/sync': specifier: workspace:* version: 'link:' @@ -1971,9 +1680,6 @@ importers: awaiting: specifier: ^3.0.0 version: 3.0.0 - debug: - specifier: ^4.4.0 - version: 4.4.1(supports-color@8.1.1) events: specifier: 3.3.0 version: 3.3.0 @@ -1990,9 +1696,6 @@ importers: specifier: ^7.18.3 version: 7.18.3 devDependencies: - '@types/debug': - specifier: ^4.1.12 - version: 4.1.12 '@types/lodash': specifier: ^4.14.202 version: 4.17.17 @@ -2028,7 +1731,7 @@ importers: version: 1.0.2 debug: specifier: ^4.4.0 - version: 4.4.1(supports-color@8.1.1) + version: 4.4.1 primus: specifier: ^8.0.9 version: 8.0.9 @@ -2042,6 +1745,9 @@ importers: '@types/debug': specifier: ^4.1.12 version: 4.1.12 + '@types/jest': + specifier: ^29.5.14 + version: 29.5.14 '@types/node': specifier: ^18.16.14 version: 18.19.111 @@ -2060,18 +1766,15 @@ importers: '@cocalc/comm': specifier: workspace:* version: link:../comm - '@cocalc/nats': + '@cocalc/conat': specifier: workspace:* - version: link:../nats + version: link:../conat '@cocalc/sync-client': specifier: workspace:* version: link:../sync-client '@cocalc/util': specifier: workspace:* version: link:../util - execa: - specifier: ^8.0.1 - version: 8.0.1 lodash: specifier: ^4.17.21 version: 4.17.21 @@ -2081,59 +1784,10 @@ importers: mkdirp: specifier: ^1.0.4 version: 1.0.4 - tsimportlib: - specifier: ^0.0.5 - version: 0.0.5 - devDependencies: - '@types/node': - specifier: ^18.16.14 - version: 18.19.111 - - terminal: - dependencies: - '@cocalc/api-client': - specifier: workspace:* - version: link:../api-client - '@cocalc/backend': - specifier: workspace:* - version: link:../backend - '@cocalc/comm': - specifier: workspace:* - version: link:../comm - '@cocalc/primus-multiplex': - specifier: ^1.1.0 - version: 1.1.0 - '@cocalc/primus-responder': - specifier: ^1.0.5 - version: 1.0.5 - '@cocalc/util': - specifier: workspace:* - version: link:../util - '@lydell/node-pty': - specifier: ^1.1.0 - version: 1.1.0 - awaiting: - specifier: ^3.0.0 - version: 3.0.0 - debug: - specifier: ^4.4.0 - version: 4.4.1(supports-color@8.1.1) - lodash: - specifier: ^4.17.21 - version: 4.17.21 - primus: - specifier: ^8.0.9 - version: 8.0.9 devDependencies: - '@types/lodash': - specifier: ^4.14.202 - version: 4.17.17 '@types/node': specifier: ^18.16.14 version: 18.19.111 - '@types/primus': - specifier: ^7.3.9 - version: 7.3.9 util: dependencies: @@ -2144,7 +1798,7 @@ importers: specifier: workspace:* version: 'link:' '@isaacs/ttlcache': - specifier: ^1.2.1 + specifier: ^1.4.1 version: 1.4.1 async: specifier: ^1.5.2 @@ -2155,9 +1809,6 @@ importers: dayjs: specifier: ^1.11.11 version: 1.11.13 - debug: - specifier: ^4.4.0 - version: 4.4.1(supports-color@8.1.1) decimal.js-light: specifier: ^2.5.1 version: 2.5.1 @@ -2216,9 +1867,6 @@ importers: specifier: ^1.3.0 version: 1.3.0 devDependencies: - '@types/debug': - specifier: ^4.1.12 - version: 4.1.12 '@types/json-stable-stringify': specifier: ^1.0.32 version: 1.2.0 @@ -2234,30 +1882,12 @@ importers: '@types/uuid': specifier: ^8.3.1 version: 8.3.4 - coffee-cache: - specifier: ^1.0.2 - version: 1.0.2 - coffee-coverage: - specifier: ^3.0.1 - version: 3.0.1 - coffeescript: - specifier: ^2.5.1 - version: 2.7.0 expect: specifier: ^26.6.2 version: 26.6.2 seedrandom: specifier: ^3.0.5 version: 3.0.5 - should: - specifier: ^7.1.1 - version: 7.1.1 - should-sinon: - specifier: 0.0.3 - version: 0.0.3(should@7.1.1) - sinon: - specifier: ^4.5.0 - version: 4.5.0 tsd: specifier: ^0.22.0 version: 0.22.0 @@ -2420,10 +2050,6 @@ packages: resolution: {integrity: sha512-BcYH1CVJBO9tvyIZ2jVeXgSIMvGZ2FDRvDdOIVQyuklNKSsx+eppDEBq/g47Ayw+RqNFE+URvOShmf+f/qwAlA==} engines: {node: '>=6.9.0'} - '@babel/code-frame@7.26.2': - resolution: {integrity: sha512-RJlIHRueQgwWitWgF8OdFYGZX328Ax5BCemNGlqHfplnRT9ESi8JkFlvaVYbS+UubVY6dpv87Fs2u5M29iNFVQ==} - engines: {node: '>=6.9.0'} - '@babel/code-frame@7.27.1': resolution: {integrity: sha512-cjQ7ZlQ0Mv3b47hABuTevyTuYN4i+loJKGeV9flcCgIK37cCXRh+L1bd3iBHlynerhQ7BhCkn2BPbQUL+rGqFg==} engines: {node: '>=6.9.0'} @@ -2436,86 +2062,32 @@ packages: resolution: {integrity: sha512-lWBYIrF7qK5+GjY5Uy+/hEgp8OJWOD/rpy74GplYRhEauvbHDeFB8t5hPOZxCZ0Oxf4Cc36tK51/l3ymJysrKw==} engines: {node: '>=6.9.0'} - '@babel/generator@7.27.0': - resolution: {integrity: sha512-VybsKvpiN1gU1sdMZIp7FcqphVVKEwcuj02x73uvcHE0PTihx1nlBcowYWhDwjpoAXRv43+gDzyggGnn1XZhVw==} - engines: {node: '>=6.9.0'} - - '@babel/generator@7.27.5': - resolution: {integrity: sha512-ZGhA37l0e/g2s1Cnzdix0O3aLYm66eF8aufiVteOgnwxgnRP8GoyMj7VWsgWnQbVKXyge7hqrFh2K2TQM6t1Hw==} - engines: {node: '>=6.9.0'} - - '@babel/helper-annotate-as-pure@7.27.3': - resolution: {integrity: sha512-fXSwMQqitTGeHLBC08Eq5yXz2m37E4pJX1qAU1+2cNedz/ifv/bVXft90VeSav5nFO61EcNgwr0aJxbyPaWBPg==} + '@babel/generator@7.27.1': + resolution: {integrity: sha512-UnJfnIpc/+JO0/+KRVQNGU+y5taA5vCbwN8+azkX6beii/ZF+enZJSOKo11ZSzGJjlNfJHfQtmQT8H+9TXPG2w==} engines: {node: '>=6.9.0'} '@babel/helper-compilation-targets@7.27.0': resolution: {integrity: sha512-LVk7fbXml0H2xH34dFzKQ7TDZ2G4/rVTOrq9V+icbbadjbVxxeFeDsNHv2SrZeWoA+6ZiTyWYWtScEIW07EAcA==} engines: {node: '>=6.9.0'} - '@babel/helper-create-class-features-plugin@7.27.1': - resolution: {integrity: sha512-QwGAmuvM17btKU5VqXfb+Giw4JcN0hjuufz3DYnpeVDvZLAObloM77bhMXiqry3Iio+Ai4phVRDwl6WU10+r5A==} - engines: {node: '>=6.9.0'} - peerDependencies: - '@babel/core': ^7.0.0 - - '@babel/helper-member-expression-to-functions@7.27.1': - resolution: {integrity: sha512-E5chM8eWjTp/aNoVpcbfM7mLxu9XGLWYise2eBKGQomAk/Mb4XoxyqXTZbuTohbsl8EKqdlMhnDI2CCLfcs9wA==} - engines: {node: '>=6.9.0'} - - '@babel/helper-module-imports@7.25.9': - resolution: {integrity: sha512-tnUA4RsrmflIM6W6RFTLFSXITtl0wKjgpnLgXyowocVPrbYrLUXSBXDgTs8BlbmIzIdlBySRQjINYs2BAkiLtw==} - engines: {node: '>=6.9.0'} - '@babel/helper-module-imports@7.27.1': resolution: {integrity: sha512-0gSFWUPNXNopqtIPQvlD5WgXYI5GY2kP2cCvoT8kczjbfcfuIljTbcWrulD1CIPIX2gt1wghbDy08yE1p+/r3w==} engines: {node: '>=6.9.0'} - '@babel/helper-module-transforms@7.26.0': - resolution: {integrity: sha512-xO+xu6B5K2czEnQye6BHA7DolFFmS3LB7stHZFaOLb1pAwO1HWLS8fXA+eh0A2yIvltPVmx3eNNDBJA2SLHXFw==} - engines: {node: '>=6.9.0'} - peerDependencies: - '@babel/core': ^7.0.0 - - '@babel/helper-module-transforms@7.27.3': - resolution: {integrity: sha512-dSOvYwvyLsWBeIRyOeHXp5vPj5l1I011r52FM1+r1jCERv+aFXYk4whgQccYEGYxK2H3ZAIA8nuPkQ0HaUo3qg==} + '@babel/helper-module-transforms@7.27.1': + resolution: {integrity: sha512-9yHn519/8KvTU5BjTVEEeIM3w9/2yXNKoD82JifINImhpKkARMJKPP59kLo+BafpdN5zgNeIcS4jsGDmd3l58g==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0 - '@babel/helper-optimise-call-expression@7.27.1': - resolution: {integrity: sha512-URMGH08NzYFhubNSGJrpUEphGKQwMQYBySzat5cAByY1/YgIRkULnIy3tAMeszlL/so2HbeilYloUmSpd7GdVw==} - engines: {node: '>=6.9.0'} - '@babel/helper-plugin-utils@7.27.1': resolution: {integrity: sha512-1gn1Up5YXka3YYAHGKpbideQ5Yjf1tDa9qYcgysz+cNCXukyLl6DjPXhD3VRwSb8c0J9tA4b2+rHEZtc6R0tlw==} engines: {node: '>=6.9.0'} - '@babel/helper-replace-supers@7.27.1': - resolution: {integrity: sha512-7EHz6qDZc8RYS5ElPoShMheWvEgERonFCs7IAonWLLUTXW59DP14bCZt89/GKyreYn8g3S83m21FelHKbeDCKA==} - engines: {node: '>=6.9.0'} - peerDependencies: - '@babel/core': ^7.0.0 - - '@babel/helper-skip-transparent-expression-wrappers@7.27.1': - resolution: {integrity: sha512-Tub4ZKEXqbPjXgWLl2+3JpQAYBJ8+ikpQ2Ocj/q/r0LwE3UhENh7EUabyHjz2kCEsrRY83ew2DQdHluuiDQFzg==} - engines: {node: '>=6.9.0'} - - '@babel/helper-string-parser@7.25.9': - resolution: {integrity: sha512-4A/SCr/2KLd5jrtOMFzaKjVtAei3+2r/NChoBNoZ3EyP/+GlhoaEGoWOZUmFmoITP7zOJyHIMm+DYRd8o3PvHA==} - engines: {node: '>=6.9.0'} - '@babel/helper-string-parser@7.27.1': resolution: {integrity: sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA==} engines: {node: '>=6.9.0'} - '@babel/helper-validator-identifier@7.24.7': - resolution: {integrity: sha512-rR+PBcQ1SMQDDyF6X0wxtG8QyLCgUB0eRAGguqRLfkCA87l7yAP7ehq8SNj96OOGTO8OBV70KhuFYcIkHXOg0w==} - engines: {node: '>=6.9.0'} - - '@babel/helper-validator-identifier@7.25.9': - resolution: {integrity: sha512-Ed61U6XJc3CVRfkERJWDz4dJwKe7iLmmJsbOGu9wSloNSFttHV0I8g6UAgb7qnK5ly5bGLPd4oXZlxCdANBOWQ==} - engines: {node: '>=6.9.0'} - '@babel/helper-validator-identifier@7.27.1': resolution: {integrity: sha512-D2hP9eA+Sqx1kBZgzxZh0y1trbuU+JoDkiEwqhQ36nodYqJwyEIhPSdMNd7lOm/4io72luTPWH20Yda0xOuUow==} engines: {node: '>=6.9.0'} @@ -2528,17 +2100,12 @@ packages: resolution: {integrity: sha512-U5eyP/CTFPuNE3qk+WZMxFkp/4zUzdceQlfzf7DdGdhp+Fezd7HD+i8Y24ZuTMKX3wQBld449jijbGq6OdGNQg==} engines: {node: '>=6.9.0'} - '@babel/highlight@7.24.7': - resolution: {integrity: sha512-EStJpq4OuY8xYfhGVXngigBJRWxftKX9ksiGDnmlY3o7B/V7KIAc9X4oiK87uPJSc/vs5L869bem5fhZa8caZw==} + '@babel/highlight@7.25.7': + resolution: {integrity: sha512-iYyACpW3iW8Fw+ZybQK+drQre+ns/tKpXbNESfrhNnPLIklLbXr7MYJ6gPEd0iETGLOK+SxMjVvKb/ffmk+FEw==} engines: {node: '>=6.9.0'} - '@babel/parser@7.27.0': - resolution: {integrity: sha512-iaepho73/2Pz7w2eMS0Q5f83+0RKI7i4xmiYeBmDzfRVbQtTOG7Ts0S4HzJVsTMGI9keU8rNfuZr8DKfSt7Yyg==} - engines: {node: '>=6.0.0'} - hasBin: true - - '@babel/parser@7.27.5': - resolution: {integrity: sha512-OsQd175SxWkGlzbny8J3K8TnnDD0N3lrIUtB92xwyRpzaenGZhxDvxN/JgU00U3CDZNj9tPuDJ5H0WS4Nt3vKg==} + '@babel/parser@7.27.2': + resolution: {integrity: sha512-QYLs8299NA7WM/bZAdp+CviYYkVoYXlDW2rzliy3chxd1PQjej7JORuMJDJXJUb9g0TT+B99EwaVLKmX+sPXWw==} engines: {node: '>=6.0.0'} hasBin: true @@ -2593,48 +2160,24 @@ packages: peerDependencies: '@babel/core': ^7.0.0-0 - '@babel/plugin-syntax-optional-catch-binding@7.8.3': - resolution: {integrity: sha512-6VPD0Pc1lpTqw0aKoeRTMiB+kWhAoT24PA+ksWSBrFtl5SIRVpZlwN3NNPQjehA2E/91FV3RjLWoVTglWcSV3Q==} - peerDependencies: - '@babel/core': ^7.0.0-0 - - '@babel/plugin-syntax-optional-chaining@7.8.3': - resolution: {integrity: sha512-KoK9ErH1MBlCPxV0VANkXW2/dw4vlbGDrFgz8bmUsBGYkFRcbRwMh6cIJubdPrkxRwuGdtCk0v/wPTKbQgBjkg==} - peerDependencies: - '@babel/core': ^7.0.0-0 - - '@babel/plugin-syntax-top-level-await@7.14.5': - resolution: {integrity: sha512-hx++upLv5U1rgYfwe1xBQUhRmU41NEvpUvrp8jkrSCdvGSnM5/qdRMtylJ6PG5OFkBaHkbTAKTnd3/YyESRHFw==} - engines: {node: '>=6.9.0'} - peerDependencies: - '@babel/core': ^7.0.0-0 - - '@babel/plugin-syntax-typescript@7.25.9': - resolution: {integrity: sha512-hjMgRy5hb8uJJjUcdWunWVcoi9bGpJp8p5Ol1229PoN6aytsLwNMgmdftO23wnCLMfVmTwZDWMPNq/D1SY60JQ==} - engines: {node: '>=6.9.0'} - peerDependencies: - '@babel/core': ^7.0.0-0 - - '@babel/plugin-syntax-typescript@7.27.1': - resolution: {integrity: sha512-xfYCBMxveHrRMnAWl1ZlPXOZjzkN82THFvLhQhFXFt81Z5HnN+EtUkZhv/zcKpmT3fzmWZB0ywiBrbC3vogbwQ==} - engines: {node: '>=6.9.0'} + '@babel/plugin-syntax-optional-catch-binding@7.8.3': + resolution: {integrity: sha512-6VPD0Pc1lpTqw0aKoeRTMiB+kWhAoT24PA+ksWSBrFtl5SIRVpZlwN3NNPQjehA2E/91FV3RjLWoVTglWcSV3Q==} peerDependencies: '@babel/core': ^7.0.0-0 - '@babel/plugin-transform-modules-commonjs@7.27.1': - resolution: {integrity: sha512-OJguuwlTYlN0gBZFRPqwOGNWssZjfIUdS7HMYtN8c1KmwpwHFBwTeFZrg9XZa+DFTitWOW5iTAG7tyCUPsCCyw==} - engines: {node: '>=6.9.0'} + '@babel/plugin-syntax-optional-chaining@7.8.3': + resolution: {integrity: sha512-KoK9ErH1MBlCPxV0VANkXW2/dw4vlbGDrFgz8bmUsBGYkFRcbRwMh6cIJubdPrkxRwuGdtCk0v/wPTKbQgBjkg==} peerDependencies: '@babel/core': ^7.0.0-0 - '@babel/plugin-transform-typescript@7.27.1': - resolution: {integrity: sha512-Q5sT5+O4QUebHdbwKedFBEwRLb02zJ7r4A5Gg2hUoLuU3FjdMcyqcywqUrLCaDsFCxzokf7u9kuy7qz51YUuAg==} + '@babel/plugin-syntax-top-level-await@7.14.5': + resolution: {integrity: sha512-hx++upLv5U1rgYfwe1xBQUhRmU41NEvpUvrp8jkrSCdvGSnM5/qdRMtylJ6PG5OFkBaHkbTAKTnd3/YyESRHFw==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 - '@babel/preset-typescript@7.27.1': - resolution: {integrity: sha512-l7WfQfX0WK4M0v2RudjuQK4u99BS6yLHYEmdtVPP7lKV013zr9DygFuWNlnbvQ9LR+LS0Egz/XAvGx5U9MX0fQ==} + '@babel/plugin-syntax-typescript@7.27.1': + resolution: {integrity: sha512-xfYCBMxveHrRMnAWl1ZlPXOZjzkN82THFvLhQhFXFt81Z5HnN+EtUkZhv/zcKpmT3fzmWZB0ywiBrbC3vogbwQ==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 @@ -2647,32 +2190,24 @@ packages: resolution: {integrity: sha512-VtPOkrdPHZsKc/clNqyi9WUA8TINkZ4cGk63UUE3u4pmB2k+ZMQRDuIOagv8UVd6j7k0T3+RRIb7beKTebNbcw==} engines: {node: '>=6.9.0'} - '@babel/runtime@7.27.6': - resolution: {integrity: sha512-vbavdySgbTTrmFE+EsiqUTzlOr5bzlnJtUv9PynGCAKvfQqjIXbvFdumPM/GxMDfyuGMJaJAU6TO4zc1Jf1i8Q==} + '@babel/runtime@7.27.1': + resolution: {integrity: sha512-1x3D2xEk2fRo3PAhwQwu5UubzgiVWSXTBfWpVd2Mx2AzRqJuDJCsgaDVZ7HB5iGzDW1Hl1sWN2mFyKjmR9uAog==} engines: {node: '>=6.9.0'} - '@babel/template@7.27.0': - resolution: {integrity: sha512-2ncevenBqXI6qRMukPlXwHKHchC7RyMuu4xv5JBXRfOGVcTy1mXCD12qrp7Jsoxll1EV3+9sE4GugBVRjT2jFA==} + '@babel/runtime@7.27.6': + resolution: {integrity: sha512-vbavdySgbTTrmFE+EsiqUTzlOr5bzlnJtUv9PynGCAKvfQqjIXbvFdumPM/GxMDfyuGMJaJAU6TO4zc1Jf1i8Q==} engines: {node: '>=6.9.0'} '@babel/template@7.27.2': resolution: {integrity: sha512-LPDZ85aEJyYSd18/DkjNh4/y1ntkE5KwUHWTiqgRxruuZL2F1yuHligVHLvcHY2vMHXttKFpJn6LwfI7cw7ODw==} engines: {node: '>=6.9.0'} - '@babel/traverse@7.27.0': - resolution: {integrity: sha512-19lYZFzYVQkkHkl4Cy4WrAVcqBkgvV2YM2TU3xG6DIwO7O3ecbDPfW3yM3bjAGcqcQHi+CCtjMR3dIEHxsd6bA==} - engines: {node: '>=6.9.0'} - - '@babel/traverse@7.27.4': - resolution: {integrity: sha512-oNcu2QbHqts9BtOWJosOVJapWjBDSxGCpFvikNR5TGDYDQf3JwpIoMzIKrvfoti93cLfPJEG4tH9SPVeyCGgdA==} - engines: {node: '>=6.9.0'} - - '@babel/types@7.27.0': - resolution: {integrity: sha512-H45s8fVLYjbhFH62dIJ3WtmJ6RSPt/3DRO0ZcT2SUiYiQyz3BLVb9ADEnLl91m74aQPS3AzzeajZHYOalWe3bg==} + '@babel/traverse@7.27.1': + resolution: {integrity: sha512-ZCYtZciz1IWJB4U61UPu4KEaqyfj+r5T1Q5mqPo+IBpcG9kHv30Z0aD8LXPgC1trYa6rK0orRyAhqUgk4MjmEg==} engines: {node: '>=6.9.0'} - '@babel/types@7.27.6': - resolution: {integrity: sha512-ETyHEk2VHHvl9b9jZP5IHPavHYk57EhanlRRuae9XCpb/j5bDCbPPMOBfCWhnl/7EDJz0jEMCi/RhccCE8r1+Q==} + '@babel/types@7.27.1': + resolution: {integrity: sha512-+EzkxvLNfiUeKMgy/3luqfsCWFRXLb7U6wNQTk60tovuckwB15B191tJWvpp4HjiQWdJkCxO3Wbvc6jlk3Xb2Q==} engines: {node: '>=6.9.0'} '@bcoe/v8-coverage@0.2.3': @@ -2681,6 +2216,13 @@ packages: '@braintree/sanitize-url@7.1.1': resolution: {integrity: sha512-i1L7noDNxtFyL5DmZafWy1wRVhGehQmzZaz1HiN5e7iylJMSZR7ekOV7NsIqa5qBldlLrsKv4HbgFUVlQrz8Mw==} + '@cfaester/enzyme-adapter-react-18@0.8.0': + resolution: {integrity: sha512-3Z3ThTUouHwz8oIyhTYQljEMNRFtlVyc3VOOHCbxs47U6cnXs8K9ygi/c1tv49s7MBlTXeIcuN+Ttd9aPtILFQ==} + peerDependencies: + enzyme: ^3.11.0 + react: '>=18' + react-dom: '>=18' + '@cfworker/json-schema@4.1.1': resolution: {integrity: sha512-gAmrUZSGtKc3AiBL71iNWxDsyUC5uMaKKGdvzYsBoTW/xi42JQHl7eKV2OYzCUqvc+D2RCcf7EXY2iCyFIk6og==} @@ -2717,6 +2259,12 @@ packages: resolution: {integrity: sha512-nFHqPo9zxbPNRkMzFH/ZbVrrp3RGb0vAVWSRG67+j8dwt3MN9pqZSEUOfWUt9AURsRJd9IR+gLnwrczQTNu6Zg==} engines: {node: '>=0.10.28'} + '@cocalc/redis-streams-adapter@0.2.3': + resolution: {integrity: sha512-9ttOz86edQS2RYt9ADN1l5s2kOnOG7d1wA11TRTrxOjaXa7xvTqgLjwzQL1LnVJ8C1+CipN8mDzw3PaSug3kWg==} + engines: {node: '>=14.0.0'} + peerDependencies: + socket.io-adapter: ^2.5.4 + '@cocalc/widgets@1.2.0': resolution: {integrity: sha512-q1Ka84hQYwocvoS81gjlgtT6cvgrEtgP9vKbAp6AzKd9moW9r6oHkduL8i9CT8GD/4b7fTJ6oAAqxh160VUuPA==} @@ -2789,14 +2337,14 @@ packages: '@cspell/dict-elixir@4.0.7': resolution: {integrity: sha512-MAUqlMw73mgtSdxvbAvyRlvc3bYnrDqXQrx5K9SwW8F7fRYf9V4vWYFULh+UWwwkqkhX9w03ZqFYRTdkFku6uA==} - '@cspell/dict-en-common-misspellings@2.1.0': - resolution: {integrity: sha512-81NUjPIH+nvNIHCRbbMVSqPPLQUqidF/l8JdlY4OFO0W253yDIk1zaZJpJ8crwYRhOLBVBnUUfm7KYx9F2V7Zg==} + '@cspell/dict-en-common-misspellings@2.0.11': + resolution: {integrity: sha512-xFQjeg0wFHh9sFhshpJ+5BzWR1m9Vu8pD0CGPkwZLK9oii8AD8RXNchabLKy/O5VTLwyqPOi9qpyp1cxm3US4Q==} '@cspell/dict-en-gb@1.1.33': resolution: {integrity: sha512-tKSSUf9BJEV+GJQAYGw5e+ouhEe2ZXE620S7BLKe3ZmpnjlNG9JqlnaBhkIMxKnNFkLY2BP/EARzw31AZnOv4g==} - '@cspell/dict-en_us@4.4.11': - resolution: {integrity: sha512-ls3ASwIL0uuAEXsxB7NsIe6GRBQ+NZfqI5k1qtNgOZ1eh1MFYjCiF+YcqArH5SFHNzOwCHRKzlLeX0ZFIok7GQ==} + '@cspell/dict-en_us@4.4.9': + resolution: {integrity: sha512-5gjqpUwhE+qP9A9wxD1+MGGJ3DNqTgSpiOsS10cGJfV4p/Z194XkDUZrUrJsnJA/3fsCZHAzcNWh8m0bw1v++A==} '@cspell/dict-filetypes@3.0.12': resolution: {integrity: sha512-+ds5wgNdlUxuJvhg8A1TjuSpalDFGCh7SkANCWvIplg6QZPXL4j83lqxP7PgjHpx7PsBUS7vw0aiHPjZy9BItw==} @@ -2816,11 +2364,11 @@ packages: '@cspell/dict-gaming-terms@1.1.1': resolution: {integrity: sha512-tb8GFxjTLDQstkJcJ90lDqF4rKKlMUKs5/ewePN9P+PYRSehqDpLI5S5meOfPit8LGszeOrjUdBQ4zXo7NpMyQ==} - '@cspell/dict-git@3.0.6': - resolution: {integrity: sha512-nazfOqyxlBOQGgcur9ssEOEQCEZkH8vXfQe8SDEx8sCN/g0SFm8ktabgLVmBOXjy3RzjVNLlM2nBfRQ7e6+5hQ==} + '@cspell/dict-git@3.0.5': + resolution: {integrity: sha512-I7l86J2nOcpBY0OcwXLTGMbcXbEE7nxZme9DmYKrNgmt35fcLu+WKaiXW7P29V+lIXjJo/wKrEDY+wUEwVuABQ==} - '@cspell/dict-golang@6.0.22': - resolution: {integrity: sha512-FvV0m3Y0nUFxw36uDCD8UtfOPv4wsZnnlabNwB3xNZ2IBn0gBURuMUZywScb9sd2wXM8VFBRoU//tc6NQsOVOg==} + '@cspell/dict-golang@6.0.21': + resolution: {integrity: sha512-D3wG1MWhFx54ySFJ00CS1MVjR4UiBVsOWGIjJ5Av+HamnguqEshxbF9mvy+BX0KqzdLVzwFkoLBs8QeOID56HA==} '@cspell/dict-google@1.0.8': resolution: {integrity: sha512-BnMHgcEeaLyloPmBs8phCqprI+4r2Jb8rni011A8hE+7FNk7FmLE3kiwxLFrcZnnb7eqM0agW4zUaNoB0P+z8A==} @@ -2840,8 +2388,8 @@ packages: '@cspell/dict-julia@1.1.0': resolution: {integrity: sha512-CPUiesiXwy3HRoBR3joUseTZ9giFPCydSKu2rkh6I2nVjXnl5vFHzOMLXpbF4HQ1tH2CNfnDbUndxD+I+7eL9w==} - '@cspell/dict-k8s@1.0.11': - resolution: {integrity: sha512-8ojNwB5j4PfZ1Gq9n5c/HKJCtZD3h6+wFy+zpALpDWFFQ2qT22Be30+3PVd+G5gng8or0LeK8VgKKd0l1uKPTA==} + '@cspell/dict-k8s@1.0.10': + resolution: {integrity: sha512-313haTrX9prep1yWO7N6Xw4D6tvUJ0Xsx+YhCP+5YrrcIKoEw5Rtlg8R4PPzLqe6zibw6aJ+Eqq+y76Vx5BZkw==} '@cspell/dict-kotlin@1.1.0': resolution: {integrity: sha512-vySaVw6atY7LdwvstQowSbdxjXG6jDhjkWVWSjg1XsUckyzH1JRHXe9VahZz1i7dpoFEUOWQrhIe5B9482UyJQ==} @@ -2858,13 +2406,13 @@ packages: '@cspell/dict-makefile@1.0.4': resolution: {integrity: sha512-E4hG/c0ekPqUBvlkrVvzSoAA+SsDA9bLi4xSV3AXHTVru7Y2bVVGMPtpfF+fI3zTkww/jwinprcU1LSohI3ylw==} - '@cspell/dict-markdown@2.0.11': - resolution: {integrity: sha512-stZieFKJyMQbzKTVoalSx2QqCpB0j8nPJF/5x+sBnDIWgMC65jp8Wil+jccWh9/vnUVukP3Ejewven5NC7SWuQ==} + '@cspell/dict-markdown@2.0.10': + resolution: {integrity: sha512-vtVa6L/84F9sTjclTYDkWJF/Vx2c5xzxBKkQp+CEFlxOF2SYgm+RSoEvAvg5vj4N5kuqR4350ZlY3zl2eA3MXw==} peerDependencies: '@cspell/dict-css': ^4.0.17 '@cspell/dict-html': ^4.0.11 '@cspell/dict-html-symbol-entities': ^4.0.3 - '@cspell/dict-typescript': ^3.2.2 + '@cspell/dict-typescript': ^3.2.1 '@cspell/dict-monkeyc@1.0.10': resolution: {integrity: sha512-7RTGyKsTIIVqzbvOtAu6Z/lwwxjGRtY5RkKPlXKHEoEAgIXwfDxb5EkVwzGQwQr8hF/D3HrdYbRT8MFBfsueZw==} @@ -2872,8 +2420,8 @@ packages: '@cspell/dict-node@5.0.7': resolution: {integrity: sha512-ZaPpBsHGQCqUyFPKLyCNUH2qzolDRm1/901IO8e7btk7bEDF56DN82VD43gPvD4HWz3yLs/WkcLa01KYAJpnOw==} - '@cspell/dict-npm@5.2.6': - resolution: {integrity: sha512-VGEY1ZjE8c8JCA+dic1IdYmVTNfVtWAw7V2n4TXO1+mKfRL+BsPsqEoH8iR0OMutC9QXjVNh32rzMh4D3E+Lxw==} + '@cspell/dict-npm@5.2.4': + resolution: {integrity: sha512-/hK5ii9OzSOQkmTjkzJlEYWz+PBnz2hRq5Xu7d4aDURaynO9xMAcK31JJlKNQulBkVbQHxFZLUrzjdzdAr/Opw==} '@cspell/dict-php@4.0.14': resolution: {integrity: sha512-7zur8pyncYZglxNmqsRycOZ6inpDoVd4yFfz1pQRe5xaRWMiK3Km4n0/X/1YMWhh3e3Sl/fQg5Axb2hlN68t1g==} @@ -2902,8 +2450,8 @@ packages: '@cspell/dict-shell@1.1.0': resolution: {integrity: sha512-D/xHXX7T37BJxNRf5JJHsvziFDvh23IF/KvkZXNSh8VqcRdod3BAz9VGHZf6VDqcZXr1VRqIYR3mQ8DSvs3AVQ==} - '@cspell/dict-software-terms@5.1.0': - resolution: {integrity: sha512-8zsOVzcHpb4PAaKtOWAIJRbpaNINaUZRsHzqFb3K9hQIC6hxmet/avLlCeKdnmBVZkn3TmRN5caxTJamJvbXww==} + '@cspell/dict-software-terms@5.0.10': + resolution: {integrity: sha512-2nTcVKTYJKU5GzeviXGPtRRC9d23MtfpD4PM4pLSzl29/5nx5MxOUHkzPuJdyaw9mXIz8Rm9IlGeVAvQoTI8aw==} '@cspell/dict-sql@2.2.0': resolution: {integrity: sha512-MUop+d1AHSzXpBvQgQkCiok8Ejzb+nrzyG16E8TvKL2MQeDwnIvMe3bv90eukP6E1HWb+V/MA/4pnq0pcJWKqQ==} @@ -3053,10 +2601,6 @@ packages: typescript: optional: true - '@google-ai/generativelanguage@3.2.0': - resolution: {integrity: sha512-/dmsV7GHx8VwhR6LI/PA4HvBckOzZNnkskYVddzvy8syc+sX2k3lq7+TRQbq2/tdmevNIGpftEg5CzAwt6zmtA==} - engines: {node: '>=18'} - '@google-cloud/bigquery@7.9.4': resolution: {integrity: sha512-C7jeI+9lnCDYK3cRDujcBsPgiwshWKn/f0BiaJmClplfyosCLfWE83iGQ0eKH113UZzjR9c9q7aZQg0nU388sw==} engines: {node: '>=14.0.0'} @@ -3138,6 +2682,9 @@ packages: peerDependencies: react: '*' + '@iovalkey/commands@0.1.0': + resolution: {integrity: sha512-/B9W4qKSSITDii5nkBCHyPkIkAi+ealUtr1oqBJsLxjSRLka4pxun2VvMNSmcwgAMxgXtQfl0qRv7TE+udPJzg==} + '@isaacs/cliui@8.0.2': resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==} engines: {node: '>=12'} @@ -3224,10 +2771,6 @@ packages: resolution: {integrity: sha512-u3UPsIilWKOM3F9CXtrG8LEJmNxwoCQC/XVj4IKYXvvpx7QIi/Kg1LI5uDmDpKlac62NUtX7eLjRh+jVZcLOzw==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} - '@jridgewell/gen-mapping@0.3.5': - resolution: {integrity: sha512-IzL8ZoEDIBRWEzlCcRhOaCupYyN5gdIK+Q6fbFdPDg6HqX6jpkItn7DFIpW9LQzXG6Df9sA7+OKnq0qlz/GaQg==} - engines: {node: '>=6.0.0'} - '@jridgewell/gen-mapping@0.3.8': resolution: {integrity: sha512-imAbBGkb+ebQyxKgzv5Hu2nmROxoDOXHh80evxdoXNOrvAnVx7zimzc1Oo5h9RlfV4vPXaE2iM5pOFbvOCClWA==} engines: {node: '>=6.0.0'} @@ -3315,8 +2858,8 @@ packages: resolution: {integrity: sha512-HLkOtVofgBHefaUae/+2fLNkpMLzEjHSavTmUF0YC7bDa5NPIZGlP80CGrSFXAeJ+WCPd8rIK8K/p6AW94inUQ==} engines: {node: '>=18'} - '@langchain/google-genai@0.2.11': - resolution: {integrity: sha512-MVQl4wrAdr35+G+P8NBC76fU+aunA5BZNRy7cl1UdHtcMPYBFufcjR7FYgGmdPgV89WP7u/V0jtv8drBs3lPCw==} + '@langchain/google-genai@0.2.12': + resolution: {integrity: sha512-dVfkNW3uJ2Ing4KAYPTfkdTDIA4FDrik/YK5A5bE8w7drP0J+7g0h2i8D5Mn7dDcyPuX74qd2VG9hWCyGLtZiw==} engines: {node: '>=18'} peerDependencies: '@langchain/core': ^0.3.46 @@ -3531,6 +3074,14 @@ packages: '@module-federation/webpack-bundler-runtime@0.14.3': resolution: {integrity: sha512-hIyJFu34P7bY2NeMIUHAS/mYUHEY71VTAsN0A0AqEJFSVPszheopu9VdXq0VDLrP9KQfuXT8SDxeYeJXyj0mgA==} + '@msgpack/msgpack@2.8.0': + resolution: {integrity: sha512-h9u4u/jiIRKbq25PM+zymTyW6bhTzELvOoUd+AvYriWOAKpLGnIamaET3pnHYoI5iYphAHBI4ayx0MehR+VVPQ==} + engines: {node: '>= 10'} + + '@msgpack/msgpack@3.1.2': + resolution: {integrity: sha512-JEW4DEtBzfe8HvUYecLU9e6+XJnKDlUAIve8FvPzF3Kzs6Xo/KuZkZJsDH0wJXl/qEZbeeE7edxDNY3kMs39hQ==} + engines: {node: '>= 18'} + '@napi-rs/canvas-android-arm64@0.1.69': resolution: {integrity: sha512-4icWTByY8zPvM9SelfQKf3I6kwXw0aI5drBOVrwfER5kjwXJd78FPSDSZkxDHjvIo9Q86ljl18Yr963ehA4sHQ==} engines: {node: '>= 10'} @@ -3598,33 +3149,6 @@ packages: '@napi-rs/triples@1.2.0': resolution: {integrity: sha512-HAPjR3bnCsdXBsATpDIP5WCrw0JcACwhhrwIAQhiR46n+jm+a2F8kBsfseAuWtSyQ+H3Yebt2k43B5dy+04yMA==} - '@nats-io/jetstream@3.0.0': - resolution: {integrity: sha512-pNJDQJfRmRoDsBYSysvk+kPr9cSj1XnLnAxSC7KHVTsu2YqMEWSqgs5YzIULbq1S12oFhM/vvlOiEj/9jY39lg==} - - '@nats-io/jwt@0.0.10-5': - resolution: {integrity: sha512-DoBIEJIgWZJLtMRef+oFvk1lEgyTIVcAZbBXYGm91B7az1bCu7A3Npsn/VD7RH5waYAz+EPS8Ot7zyU6iKAuqg==} - - '@nats-io/kv@3.0.0': - resolution: {integrity: sha512-ybERTWMemaBpMdbFv4718Ffnb427ykzlci+6oAmtaeyw3oM25aACEo+xgra9UKxQVubj1UGx2WcNUy9ojiU9zA==} - - '@nats-io/nats-core@3.0.0': - resolution: {integrity: sha512-Ma2VrFkSew35cIRxMvnbQ2VOE2crP7BCIIsGoCqVwvn6Jwmk2ypcHjj9pWn+bPXqmFlamLUrYGH6ncMDb4eXsQ==} - - '@nats-io/nkeys@1.2.0-8': - resolution: {integrity: sha512-o6nfNYySzVZL7mIJA+ejD3JdMFbLn9VWtptM2cnHh4jeF/FvhPhCkn8BH4jQE7asCV/SSialu6jGhsAFV1FKkQ==} - engines: {node: '>=16.0.0'} - - '@nats-io/nkeys@2.0.3': - resolution: {integrity: sha512-JVt56GuE6Z89KUkI4TXUbSI9fmIfAmk6PMPknijmuL72GcD+UgIomTcRWiNvvJKxA01sBbmIPStqJs5cMRBC3A==} - engines: {node: '>=18.0.0'} - - '@nats-io/nuid@2.0.3': - resolution: {integrity: sha512-TpA3HEBna/qMVudy+3HZr5M3mo/L1JPofpVT4t0HkFGkz2Cn9wrlrQC8tvR8Md5Oa9//GtGG26eN0qEWF5Vqew==} - engines: {node: '>= 18.x'} - - '@nats-io/services@3.0.0': - resolution: {integrity: sha512-tsZu5Chd616p2sP9QWZT0gBkO2zcRjbENhcJcV2hBGA6t5jgP5+iynwMAs+9HKFUNVdUgDxdpUYn8NUpaslzPw==} - '@nestjs/axios@4.0.0': resolution: {integrity: sha512-1cB+Jyltu/uUPNQrpUimRHEQHrnQrpLzVj6dU3dgn6iDDDdahr10TgHFGTmw5VuJ9GzKZsCLDL78VSwJAs/9JQ==} peerDependencies: @@ -4156,29 +3680,29 @@ packages: resolution: {integrity: sha512-LtoMMhxAlorcGhmFYI+LhPgbPZCkgP6ra1YL604EeF6U98pLlQ3iWIGMdWSC+vWmPBWBNgmDBAhnAobLROJmwg==} engines: {node: '>=18'} - '@sinonjs/commons@1.8.6': - resolution: {integrity: sha512-Ky+XkAkqPZSm3NLBeUng77EBQl3cmeJhITaGHdYH8kjVB+aun3S4XBRti2zt17mtt0mIUDiNxYeoJm6drVvBJQ==} - '@sinonjs/commons@3.0.1': resolution: {integrity: sha512-K3mCHKQ9sVh8o1C9cxkwxaOmXoAMlDxC1mYyHrjqOWEcBjYr76t96zL2zlj5dUGZ3HSw240X1qgH3Mjf1yJWpQ==} '@sinonjs/fake-timers@10.3.0': resolution: {integrity: sha512-V4BG07kuYSUkTCSBHG8G8TNhM+F19jXFWnQtzj+we8DrkpSBCee9Z3Ms8yiGer/dlmhe35/Xdgyo3/0rQKg7YA==} - '@sinonjs/formatio@2.0.0': - resolution: {integrity: sha512-ls6CAMA6/5gG+O/IdsBcblvnd8qcO/l1TYoNeAzp3wcISOxlPXQEus0mLcdwazEkWjaBdaJ3TaxmNgCLWwvWzg==} - - '@sinonjs/formatio@3.2.2': - resolution: {integrity: sha512-B8SEsgd8gArBLMD6zpRw3juQ2FVSsmdd7qlevyDqzS9WTCtvF55/gAL+h6gue8ZvPYcdiPdvueM/qm//9XzyTQ==} + '@socket.io/cluster-adapter@0.2.2': + resolution: {integrity: sha512-/tNcY6qQx0BOgjl4mFk3YxX6pjaPdEyeWhP88Ea9gTlISY4SfA7t8VxbryeAs5/9QgXzChlvSN/i37Gog3kWag==} + engines: {node: '>=10.0.0'} + peerDependencies: + socket.io-adapter: ^2.4.0 - '@sinonjs/samsam@3.3.3': - resolution: {integrity: sha512-bKCMKZvWIjYD0BLGnNrxVuw4dkWCYsLqFOUWw8VgKF/+5Y+mE7LfHWPIYoDXowH+3a9LsWDMo0uAP8YDosPvHQ==} + '@socket.io/component-emitter@3.1.2': + resolution: {integrity: sha512-9BCxFwvbGg/RsZK9tjXd8s4UcwR0MWeFQ1XEKIQVVvAGJyINdrqKMcTRyLoK8Rse1GjzLV9cwjWV1olXRWEXVA==} - '@sinonjs/text-encoding@0.7.3': - resolution: {integrity: sha512-DE427ROAphMQzU4ENbliGYrBSYPXF+TtLg9S8vzeA+OF4ZKzoDdzfL8sxuMUGS/lgRhM6j1URSk9ghf7Xo1tyA==} + '@socket.io/redis-adapter@8.3.0': + resolution: {integrity: sha512-ly0cra+48hDmChxmIpnESKrc94LjRL80TEmZVscuQ/WWkRP81nNj8W8cCGMqbI4L6NCuAaPRSzZF1a9GlAxxnA==} + engines: {node: '>=10.0.0'} + peerDependencies: + socket.io-adapter: ^2.5.4 - '@speed-highlight/core@1.2.7': - resolution: {integrity: sha512-0dxmVj4gxg3Jg879kvFS/msl4s9F3T9UXC1InxgOf7t5NvcPD97u/WTA5vL/IxWHMn7qSxBozqrnnE2wvl1m8g==} + '@socket.io/sticky@1.0.4': + resolution: {integrity: sha512-VuauT5CJLvzYtKIgouFSQ8rUaygseR+zRutnwh6ZA2QYcXx+8g52EoJ8V2SLxfo+Tfs3ELUDy08oEXxlWNrxaw==} '@stripe/react-stripe-js@3.7.0': resolution: {integrity: sha512-PYls/2S9l0FF+2n0wHaEJsEU8x7CmBagiH7zYOsxbBlLIHEsqUIQ4MlIAbV9Zg6xwT8jlYdlRIyBTHmO3yM7kQ==} @@ -4287,6 +3811,9 @@ packages: '@types/cookie@0.6.0': resolution: {integrity: sha512-4Kh9a6B2bQciAhf7FSuMRRkUWecJgJu9nPnx3yzpsfXX/c50REIqpHY4C82bXP90qrLtXtkDxTZosYO3UpOwlA==} + '@types/cors@2.8.18': + resolution: {integrity: sha512-nX3d0sxJW41CqQvfOzVG1NCTXfFDrDWIghCZncpHeWlVFd81zxB/DLhg7avFg6eHLCRX7ckBmoIIcqa++upvJA==} + '@types/d3-array@3.2.1': resolution: {integrity: sha512-Y2Jn2idRrLzUfAKV2LyRImR+y4oa2AntrgID95SHJxuMUrkNXmanDSed71sRNZysveJVt1hLLemQZIady0FpEg==} @@ -4431,6 +3958,9 @@ packages: '@types/hast@2.3.10': resolution: {integrity: sha512-McWspRw8xx8J9HurkVBfYj0xKoE25tOFlHGdx4MJ5xORQrMGZNqJhVQWaIbm6Oyla5kYOXtDiopzKRJzEOkwJw==} + '@types/hast@3.0.4': + resolution: {integrity: sha512-WPs+bbQw5aCj+x6laNGWLH3wviHtoCv/P3+otBhbOhJgG8qtpdAMlTCxLtsTWA7LH1Oh/bFCHsBn0TPS5m30EQ==} + '@types/hoist-non-react-statics@3.3.1': resolution: {integrity: sha512-iMIqiko6ooLrTh1joXodJK5X9xeEALT1kM5G3ZLhD3hszxBdIEd5C75U834D9mLcINgD4OyZf5uQXjkuYydWvA==} @@ -4477,9 +4007,6 @@ packages: '@types/keyv@3.1.4': resolution: {integrity: sha512-BQ5aZNSCpj7D6K2ksrRCTmKRLEpnPvWDiLPfoGyhZ++8YtiK9d/3DBKPJgry359X/P1PfruyYwvnvwFjuEiEIg==} - '@types/ldapjs@2.2.5': - resolution: {integrity: sha512-Lv/nD6QDCmcT+V1vaTRnEKE8UgOilVv5pHcQuzkU1LcRe4mbHHuUo/KHi0LKrpdHhQY8FJzryF38fcVdeUIrzg==} - '@types/linkify-it@5.0.0': resolution: {integrity: sha512-sVDA58zAw4eWAffKOaQH5/5j3XeayukzDk+ewSsnv3p4yJEZHCCzMDiZM8e0OUrRvmpGZ85jf4yDHkHsgBNr9Q==} @@ -4489,10 +4016,6 @@ packages: '@types/long@4.0.2': resolution: {integrity: sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA==} - '@types/long@5.0.0': - resolution: {integrity: sha512-eQs9RsucA/LNjnMoJvWG/nXa7Pot/RbBzilF/QRIU/xRl+0ApxrSUFsV5lmf01SvSlqMzJ7Zwxe440wmz2SJGA==} - deprecated: This is a stub types definition. long provides its own type definitions, so you do not need this installed. - '@types/mapbox__point-geometry@0.1.4': resolution: {integrity: sha512-mUWlSxAmYLfwnRBmgYV86tgYmMIICX4kza8YnE/eIlywGe2XoOxlpVnXWwir92xRLjwyarqwpu2EJKD2pk0IUA==} @@ -4505,6 +4028,9 @@ packages: '@types/md5@2.3.5': resolution: {integrity: sha512-/i42wjYNgE6wf0j2bcTX6kuowmdL/6PE4IVitMpm2eYKBUuYCprdcWVK+xEF0gcV6ufMCRhtxmReGfc6hIK7Jw==} + '@types/mdast@4.0.4': + resolution: {integrity: sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA==} + '@types/mdurl@2.0.0': resolution: {integrity: sha512-RGdgjQUZba5p6QEFAVx2OGb8rQDL/cPRG7GiedRzMcJ1tYnUANBncjbSB1NRGwbvjcPeikRABz2nshyPk1bhWg==} @@ -4517,15 +4043,12 @@ packages: '@types/minimist@1.2.2': resolution: {integrity: sha512-jhuKLIRrhvCPLqwPcx6INqmKeiA5EWrsCOPhrlFSrbrmU4ZMPjj5Ul/oLCMDO98XRUIwVm78xICz4EPCektzeQ==} - '@types/mocha@10.0.10': - resolution: {integrity: sha512-xPyYSz1cMPnJQhl0CLMH68j3gprKZaTjG3s5Vi+fDgx+uhG9NOXwbVt52eFS8ECyXhyKcjDLCBEqBExKuiZb7Q==} + '@types/ms@0.7.31': + resolution: {integrity: sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA==} '@types/ms@0.7.34': resolution: {integrity: sha512-nG96G3Wp6acyAgJqGasjODb+acrI7KltPiRxzHPXnP3NgI28bpQDRv53olbqGXbfcgF5aiiHmO3xpwEpS5Ld9g==} - '@types/ms@2.1.0': - resolution: {integrity: sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA==} - '@types/node-cleanup@2.1.5': resolution: {integrity: sha512-+82RAk5uYiqiMoEv2fPeh03AL4pB5d3TL+Pf+hz31Mme6ECFI1kRlgmxYjdSlHzDbJ9yLorTnKi4Op5FA54kQQ==} @@ -4544,9 +4067,6 @@ packages: '@types/node@24.0.1': resolution: {integrity: sha512-MX4Zioh39chHlDJbKmEgydJDS3tspMP/lnQC67G3SWsTnb9NeYVWOjkxpOSy4oMfPs4StcWHwBrvUb4ybfnuaw==} - '@types/node@9.6.61': - resolution: {integrity: sha512-/aKAdg5c8n468cYLy2eQrcR5k6chlbNwZNGUj3TboyPa2hcO2QAJcfymlqPzMiRj8B6nYKXjzQz36minFE0RwQ==} - '@types/nodemailer@6.4.17': resolution: {integrity: sha512-I9CCaIp6DTldEg7vyUTZi8+9Vo0hi1/T8gv3C89yk1rSAAzoKQ8H8ki/jBYJSFoH/BisgLP8tkZMlQ91CIquww==} @@ -4556,9 +4076,6 @@ packages: '@types/oauth@0.9.6': resolution: {integrity: sha512-H9TRCVKBNOhZZmyHLqFt9drPM9l+ShWiqqJijU1B8P3DX3ub84NjxDuy+Hjrz+fEca5Kwip3qPMKNyiLgNJtIA==} - '@types/parse5@6.0.3': - resolution: {integrity: sha512-SuT16Q1K51EAVPz1K29DJ/sXjhSQ0zjvsypYJ6tlwVsRV9jwW5Adq2ch8Dq8kDBCkYnELS7N7VNCSB5nC56t/g==} - '@types/passport-google-oauth20@2.0.16': resolution: {integrity: sha512-ayXK2CJ7uVieqhYOc6k/pIr5pcQxOLB6kBev+QUGS7oEZeTgIs1odDobXRqgfBPvXzl0wXCQHftV5220czZCPA==} @@ -4577,9 +4094,6 @@ packages: '@types/pg@8.15.4': resolution: {integrity: sha512-I6UNVBAoYbvuWkkU3oosC8yxqH21f4/Jc4DK71JLG3dT2mdlGe1z+ep/LQGXaKaOgcvUrsQoPRqfgtMcvZiJhg==} - '@types/pica@5.1.3': - resolution: {integrity: sha512-13SEyETRE5psd9bE0AmN+0M1tannde2fwHfLVaVIljkbL9V0OfFvKwCicyeDvVYLkmjQWEydbAlsDsmjrdyTOg==} - '@types/primus@7.3.9': resolution: {integrity: sha512-5dZ/vciGvoNxXzbDksgu3OUQ00SOQMleKVNDA7HgsB/qlhL/HayKreKCrchO+nMjGU2NHpy90g80cdzCeevxKg==} @@ -4606,9 +4120,6 @@ packages: '@types/react-redux@7.1.34': resolution: {integrity: sha512-GdFaVjEbYv4Fthm2ZLvj1VSCedV7TqE5y1kNwnjSdBOTXuRSgowux6J8TAct15T3CKBr63UMk+2CO7ilRhyrAQ==} - '@types/react@18.3.10': - resolution: {integrity: sha512-02sAAlBnP39JgXwkAq3PeU9DVaaGpZyF3MGcC0MKgQVkZor5IiiDAipVaxQHtDJAmO4GIy/rVBy/LzVj76Cyqg==} - '@types/react@18.3.23': resolution: {integrity: sha512-/LDXMQh55EzZQ0uVAZmKKhfENivEvWz6E+EYzh+/MCjMhNsotd+ZHhBGIjFDTi6+fz0OhQQQLbTgdQIxxCsC0w==} @@ -4675,6 +4186,9 @@ packages: '@types/unist@2.0.11': resolution: {integrity: sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA==} + '@types/unist@3.0.3': + resolution: {integrity: sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q==} + '@types/use-sync-external-store@0.0.3': resolution: {integrity: sha512-EwmlvuaxPNej9+T4v5AuBPJa2x2UOJVdjCtDHgcDqitUeOtjnJKJ+apYjVcAoBEMjKW1VVFGZLUb5+qqa09XFA==} @@ -4684,9 +4198,6 @@ packages: '@types/uuid@8.3.4': resolution: {integrity: sha512-c/I8ZRb51j+pYGAu5CrFMRxqZ2ke4y2grEBO5AUjgSkSk+qT2Ea+OdWElz/OiMf5MNpn2b17kuVBwZLQJXzihw==} - '@types/watchpack@2.4.4': - resolution: {integrity: sha512-SbuSavsPxfOPZwVHBgQUVuzYBe6+8KL7dwiJLXaj5rmv3DxktOMwX5WP1J6UontwUbewjVoc7pCgZvqy6rPn+A==} - '@types/ws@8.18.1': resolution: {integrity: sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==} @@ -4763,8 +4274,8 @@ packages: resolution: {integrity: sha512-JJtkDduxLi9bivAB+cYOVMtbkqdPOhZ+ZI5LC47MIRrDV4Yn2o+ZnW10Nkmr28xRpSpdJ6Sm42Hjf2+REYXm0A==} engines: {node: ^16.0.0 || >=18.0.0} - '@uiw/react-textarea-code-editor@2.1.9': - resolution: {integrity: sha512-fby8oencLyF1BMAMDVIe4zErb01Qf97G25vJld6mJmgFAbK5TwFW0XUvkxAuNKaLp+EccKf5pejCVHcS/jZ3eA==} + '@uiw/react-textarea-code-editor@3.1.1': + resolution: {integrity: sha512-AERRbp/d85vWR+UPgsB5hEgerNXuyszdmhWl2fV2H2jN63jgOobwEnjIpb76Vwy8SaGa/AdehaoJX2XZgNXtJA==} peerDependencies: '@babel/runtime': '>=7.10.0' react: '>=16.9.0' @@ -4842,6 +4353,24 @@ packages: resolution: {integrity: sha512-p96FSY54r+WJ50FIOsCOjyj/wavs8921hG5+kVMmZgKcvIKxMXHTrjNJvRgWa/zuX3B6t2lijLNFaOyuxUH+2A==} engines: {node: '>=14.6'} + '@xterm/addon-fit@0.10.0': + resolution: {integrity: sha512-UFYkDm4HUahf2lnEyHvio51TNGiLK66mqP2JoATy7hRZeXaGMRDr00JiSF7m63vR5WKATF605yEggJKsw0JpMQ==} + peerDependencies: + '@xterm/xterm': ^5.0.0 + + '@xterm/addon-web-links@0.11.0': + resolution: {integrity: sha512-nIHQ38pQI+a5kXnRaTgwqSHnX7KE6+4SVoceompgHL26unAxdfP6IPqUTSYPQgSwM56hsElfoNrrW5V7BUED/Q==} + peerDependencies: + '@xterm/xterm': ^5.0.0 + + '@xterm/addon-webgl@0.18.0': + resolution: {integrity: sha512-xCnfMBTI+/HKPdRnSOHaJDRqEpq2Ugy8LEj9GiY4J3zJObo3joylIFaMvzBwbYRg8zLtkO0KQaStCeSfoaI2/w==} + peerDependencies: + '@xterm/xterm': ^5.0.0 + + '@xterm/xterm@5.5.0': + resolution: {integrity: sha512-hqJHYaQb5OptNunnyAnkHyM8aCjZ1MEIDTQu1iIbbTD/xops91NB5yq1ZK/dC2JDbVWtF23zUtl9JE2NqwT87A==} + '@xtuc/ieee754@1.2.0': resolution: {integrity: sha512-DX8nKgqcGwsc0eJSqYt5lwP4DH5FlHnmuWWBRy7X0NcaGR0ZtuyeESgMwTYVEtxmsNGY+qit4QYT/MIYTOTPeA==} @@ -4958,10 +4487,6 @@ packages: anser@2.3.2: resolution: {integrity: sha512-PMqBCBvrOVDRqLGooQb+z+t1Q0PiPyurUQeZRR5uHBOVZcW8B04KMmnT12USnhpNX2wCPagWzLVppQMUG3u0Dw==} - ansi-colors@4.1.3: - resolution: {integrity: sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw==} - engines: {node: '>=6'} - ansi-escapes@4.3.2: resolution: {integrity: sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ==} engines: {node: '>=8'} @@ -5032,9 +4557,6 @@ packages: array-flatten@1.1.1: resolution: {integrity: sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==} - array-from@2.1.1: - resolution: {integrity: sha512-GQTc6Uupx1FCavi5mPzBvVT7nEOeWMmUA9P95wpfpW1XwMSKs+KaymD5C2Up7KAUKg/mYwbsUYzdZWcoajlNZg==} - array-includes@3.1.8: resolution: {integrity: sha512-itaWrbYbqpGXkGhZPGUulwnhVf5Hpy1xiCFsGqyIGglbBxmG5vSjxQen3/WGOjPpNEv1RtBLKxbmVXm8HpJStQ==} engines: {node: '>= 0.4'} @@ -5066,6 +4588,10 @@ packages: resolution: {integrity: sha512-MNha4BWQ6JbwhFhj03YK552f7cb3AzoE8SzeljgChvL1dl3IcvggXVz1DilzySZkCja+CXuZbdW7yATchWn8/Q==} engines: {node: '>=0.10.0'} + array.prototype.filter@1.0.4: + resolution: {integrity: sha512-r+mCJ7zXgXElgR4IRC+fkvNCeoaavWBs6EdCso5Tbcf+iEMKzBU/His60lt34WEZ9vlb8wDkZvQGcVI5GwkfoQ==} + engines: {node: '>= 0.4'} + array.prototype.findlast@1.2.5: resolution: {integrity: sha512-CVvd6FHg1Z3POpBLxO6E6zr+rSKEQ9L6rZHAaY7lLfhKsWYUBBOuMs0e9o24oopj6H+geRCX0YJ+TJLBK2eHyQ==} engines: {node: '>= 0.4'} @@ -5097,6 +4623,10 @@ packages: asap@2.0.6: resolution: {integrity: sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==} + ascii-table3@1.0.1: + resolution: {integrity: sha512-xOCMZC8S375W4JajrAxFWPyI1VddfbscW9G5zMfhCySSt2Rvi/rs21jAjopzldTPOaFrOocjyGKibQiGExmLrg==} + engines: {node: '>=11.14.0'} + asn1@0.2.6: resolution: {integrity: sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ==} @@ -5164,12 +4694,12 @@ packages: resolution: {integrity: sha512-19i4G7Hjxj9idgMlAM0BTRII8HfvsOdlr4D9cf3Dm1MZhvcKjBpzY8AMNEyIKyi+L9TIK15xZatmdcPG003yww==} engines: {node: '>=7.6.x'} + axios@1.10.0: + resolution: {integrity: sha512-/1xYAC4MP/HEG+3duIhFr4ZQXR4sQXOIe+o6sdqzeykGLx6Upp/1p8MHqhINOvGeP7xyNHe7tsiJByc4SSVUxw==} + axios@1.9.0: resolution: {integrity: sha512-re4CqKTJaURpzbLHtIi6XpDv20/CnpXOtjRY5/CU32L8gU8ek9UIivcfvSWvmKEngmVbrUtPpdDwWDWL7DNHvg==} - b4a@1.6.7: - resolution: {integrity: sha512-OnAYlL5b7LEkALw87fUVafQw5rVR9RjwGd4KUwNQ6DrrNmaVaUCgLipfVlzrPQ4tWOR9P0IXGNOx50jYCCdSJg==} - babel-jest@29.7.0: resolution: {integrity: sha512-BrvGY3xZSwEcCzKvKsCi2GgHqDqsYkOP4/by5xCgIwGXQxIEh+8ew3gmrE1y7XRR6LHZIj6yLYnUi/mm2KXKBg==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} @@ -5226,36 +4756,6 @@ packages: balanced-match@1.0.2: resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==} - bare-events@2.5.4: - resolution: {integrity: sha512-+gFfDkR8pj4/TrWCGUGWmJIkBwuxPS5F+a5yWjOHQt2hHvNZd5YLzadjmDUtFmMM4y429bnKLa8bYBMHcYdnQA==} - - bare-fs@4.1.2: - resolution: {integrity: sha512-8wSeOia5B7LwD4+h465y73KOdj5QHsbbuoUfPBi+pXgFJIPuG7SsiOdJuijWMyfid49eD+WivpfY7KT8gbAzBA==} - engines: {bare: '>=1.16.0'} - peerDependencies: - bare-buffer: '*' - peerDependenciesMeta: - bare-buffer: - optional: true - - bare-os@3.6.1: - resolution: {integrity: sha512-uaIjxokhFidJP+bmmvKSgiMzj2sV5GPHaZVAIktcxcpCyBFFWO+YlikVAdhmUo2vYFvFhOXIAlldqV29L8126g==} - engines: {bare: '>=1.14.0'} - - bare-path@3.0.0: - resolution: {integrity: sha512-tyfW2cQcB5NN8Saijrhqn0Zh7AnFNsnczRcuWODH0eYAXBsJ5gVxAUuNr7tsHSC6IZ77cA0SitzT+s47kot8Mw==} - - bare-stream@2.6.5: - resolution: {integrity: sha512-jSmxKJNJmHySi6hC42zlZnq00rga4jjxcgNZjY9N5WlOe/iOoGRtdwGsHzQv2RlH2KOYMwGUXhf2zXd32BA9RA==} - peerDependencies: - bare-buffer: '*' - bare-events: '*' - peerDependenciesMeta: - bare-buffer: - optional: true - bare-events: - optional: true - base-64@1.0.0: resolution: {integrity: sha512-kwDPIFCGx0NZHog36dj+tHiwP4QMzsZ3AgMViUBKI0+V5n4U0ufTCUMhnQ04diaRI8EX/QcPfql7zlhZ7j4zgg==} @@ -5273,6 +4773,10 @@ packages: base64-js@1.5.1: resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} + base64id@2.0.0: + resolution: {integrity: sha512-lGe34o6EHj9y3Kts9R4ZYs/Gr+6N7MCaMlIFA3F1R2O5/m7K06AxfSeO5530PEERE6/WyEg3lsuyw4GHlPZHog==} + engines: {node: ^4.5.0 || >= 5.9} + base64url@3.0.1: resolution: {integrity: sha512-ir1UPr3dkwexU7FdV8qBBbNDRUhMmIekYMFZfi+C/sLNnRESKPl23nB9b2pltqfOQNnGzsDdId90AEtG5tCx4A==} engines: {node: '>=6.0.0'} @@ -5288,9 +4792,6 @@ packages: batch@0.6.1: resolution: {integrity: sha512-x+VAiMRL6UPkx+kudNvxTl6hB2XNNCG2r+7wixVfIYwu/2HKRXimwQyaumLjMveWvT2Hkd/cAJw+QBMfJ/EKVw==} - bcryptjs@2.4.3: - resolution: {integrity: sha512-V/Hy/X9Vt7f3BbPJEi8BdVFMByHi+jNXrYkW3huaybV/kQ0KJg0Y6PkEMbn+zeT+i+SiKZ/HMqJGIIt4LZDqNQ==} - better-sqlite3@11.10.0: resolution: {integrity: sha512-EwhOpyXiOEL/lKzHz9AW1msWFNzGc/z+LzeB3/jnFJpxu+th2yqvzsSWas1v9jgs9+xiXJcD5A8CJxAG2TaghQ==} @@ -5356,9 +4857,6 @@ packages: resolution: {integrity: sha512-yN5oZVmRCwe5aKwzRj6736nSmKDX7pLYwsXiCj/EYmo16hODaBiT4En5btW/jhBF/seV+XMx3aYwukYC3A49DA==} engines: {node: '>=6'} - bottleneck@2.19.5: - resolution: {integrity: sha512-VHiNCbI1lKdl44tGrhNfU3lup0Tj/ZBMJB5/2ZbNXRCPuRCO7ed2mgcK4r17y+KB2EfuYuRaVlwNbAeaWGSpbw==} - brace-expansion@1.1.11: resolution: {integrity: sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==} @@ -5369,11 +4867,8 @@ packages: resolution: {integrity: sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==} engines: {node: '>=8'} - browser-stdout@1.3.1: - resolution: {integrity: sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw==} - - browserslist@4.24.4: - resolution: {integrity: sha512-KDi1Ny1gSePi1vm0q4oxSF8b4DR44GF4BbmS2YdhPLOEqd8pDviZOGH/GsmRwoWJ2+5Lr085X7naowMwKHDG1A==} + browserslist@4.24.5: + resolution: {integrity: sha512-FDToo4Wo82hIdgc1CQ+NQD0hEhmpPjrZ3hiUgwgOG6IuTdlpr8jdjyG24P6cNP1yJpTLzS5OcGgSw0xmDU1/Tw==} engines: {node: ^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7} hasBin: true @@ -5459,6 +4954,9 @@ packages: caniuse-lite@1.0.30001713: resolution: {integrity: sha512-wCIWIg+A4Xr7NfhTuHdX+/FKh3+Op3LBbSp2N5Pfx6T/LhdQy3GTyoTg48BReaW/MyMNZAkTadsBtai3ldWK0Q==} + caniuse-lite@1.0.30001718: + resolution: {integrity: sha512-AflseV1ahcSunK53NfEs9gFWgOEmzr0f+kaMFA4xiLZlr9Hzt7HxcSpIFcnNCUkz6R6dWKa54rUz3HUmI3nVcw==} + caniuse-lite@1.0.30001723: resolution: {integrity: sha512-1R/elMjtehrFejxwmexeXAtae5UO9iSyFn6G/I806CYC/BLyyBk1EPhrKBkWhy6wM6Xnm47dSJQec+tLJ39WHw==} @@ -5624,6 +5122,14 @@ packages: resolution: {integrity: sha512-EcR6r5a8bj6pu3ycsa/E/cKVGuTgZJZdsyUYHOksG/UHIiKfjxzRxYJpyVBwYaQeOvghal9fcc4PidlgzugAQg==} engines: {node: '>=6'} + cluster-key-slot@1.1.2: + resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==} + engines: {node: '>=0.10.0'} + + cmake-ts@0.6.1: + resolution: {integrity: sha512-uUn2qGhf20j8W/sQ7+UnvvqO1zNccqgbLgwRJi7S23FsjMWJqxvKK80Vc+tvLNKfpJzwH0rgoQD1l24SMnX0yg==} + hasBin: true + co@4.6.0: resolution: {integrity: sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==} engines: {iojs: '>= 1.0.0', node: '>= 0.12.0'} @@ -5634,10 +5140,6 @@ packages: coffee-cache@1.0.2: resolution: {integrity: sha512-sIqhtqg5AOfgVWH8uQ0b0i0rkawDD1icZNOgyYNJOlJBm8RrEzstLeOwgjZPcyVeMq0jNryx2TDL5xcPUo/27A==} - coffee-coverage@3.0.1: - resolution: {integrity: sha512-wY7ZYhxhQoG27XbJgWx2QUCi9xHrJO91+4RA7hjAWV2VVugZv3ULPXGGetI04SmFZeQ3rnZ1eFTdksi9LdEKqg==} - hasBin: true - coffee-loader@3.0.0: resolution: {integrity: sha512-2UPQNXfMAt4RmI/K9VxnLyrXdYdHPHQuEFiGcb70pTsVPmrV9M6Xg3p9ub7t1ettZZqvXUujjHozp22uTfLkzg==} engines: {node: '>= 12.13.0'} @@ -5649,12 +5151,6 @@ packages: resolution: {integrity: sha512-TTedgCCnIR978F1hwiv/aufBMz0e2pMrb8wEwf20X9VDqDvCWokTQx0ioMY/9c9eq7DnC5nb9wKGspF2IzcoFQ==} hasBin: true - coffee-script@1.12.7: - resolution: {integrity: sha512-fLeEhqwymYat/MpTPUjSKHVYYl0ec2mOyALEMLmzr5i1isuG+6jfI2j2d5oBO3VIzgUXgBVIcOT9uH1TFxBckw==} - engines: {node: '>=0.8.0'} - deprecated: CoffeeScript on NPM has moved to "coffeescript" (no hyphen) - hasBin: true - coffeescript@2.7.0: resolution: {integrity: sha512-hzWp6TUE2d/jCcN67LrW1eh5b/rSDKQK6oD6VMLlggYVUUFexgTH9z3dNYihzX4RMhze5FTUsUmOXViJKFQR/A==} engines: {node: '>=6'} @@ -5706,10 +5202,6 @@ packages: color@3.2.1: resolution: {integrity: sha512-aBl7dZI9ENN6fUGC7mWpMTPNHmWUSNan9tuWN6ahh5ZLNk9baLJOnSMlrQkHcrfFgz2/RigjUVAjdx36VcemKA==} - color@4.2.3: - resolution: {integrity: sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A==} - engines: {node: '>=12.5.0'} - colorette@2.0.20: resolution: {integrity: sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==} @@ -5809,6 +5301,9 @@ packages: connected@0.0.2: resolution: {integrity: sha512-J8DB7618GkIYjc1RCxSdG3vffhhYRwHNEckjOGfwAbabQIMgKsL5c54IaWtisulEwoOEbEODEUak4kyJP2GJ/Q==} + consistent-hash@1.2.2: + resolution: {integrity: sha512-xKAVji9aC80uN/h5u4XXwJ8otOh/D/cm8aryKa7+I0qwoBwxYOYkTiyvgwjUBRn+7CRVEvn/DvJtX1YsEugr0w==} + consola@2.15.3: resolution: {integrity: sha512-9vAdYbHj6x2fLKC4+oPH0kFzY/orMZyG2Aj+kNylHxKGJ/Ed4dpNyAQYwJOdqO4zdM7XpVHmyejQDcQHrnuXbw==} @@ -5816,8 +5311,8 @@ packages: resolution: {integrity: sha512-5IKcdX0nnYavi6G7TtOhwkYzyjfJlatbjMjuLSfE2kYT5pMDOilZ4OvMhi637CcDICTmz3wARPoyhqyX1Y+XvA==} engines: {node: ^14.18.0 || >=16.10.0} - console-table-printer@2.14.3: - resolution: {integrity: sha512-X5OCFnjYlXzRuC8ac5hPA2QflRjJvNKJocMhlnqK/Ap7q3DHXr0NJ0TGzwmEKOiOdJrjsSwEd0m+a32JAYPrKQ==} + console-table-printer@2.12.1: + resolution: {integrity: sha512-wKGOQRRvdnd89pCeH96e2Fn4wkbenSP6LMHfjfyNLMbGuHEFbMqQNuxXqd0oXG9caIOQ1FTvc5Uijp9/4jujnQ==} console.table@0.10.0: resolution: {integrity: sha512-dPyZofqggxuvSf7WXvNjuRfnsOk1YazkVP8FdxH4tcH2c37wc79/Yl6Bhr7Lsu00KMgy2ql/qCMuNu8xctZM8g==} @@ -5961,9 +5456,6 @@ packages: engines: {node: '>=18'} hasBin: true - css-color-names@0.0.4: - resolution: {integrity: sha512-zj5D7X1U2h2zsXOAM8EyUREBnnts6H+Jm+d1M2DbiQQcUtnqgQsMrdo8JW9R80YFUmIdBZeMu5wvYM7hcgWP/Q==} - css-font-size-keywords@1.0.0: resolution: {integrity: sha512-Q+svMDbMlelgCfH/RVDKtTDaf5021O486ZThQPIpahnIjUkMUslC+WuOQSWTgGSrNCH08Y7tYNEmmy0hkfMI8Q==} @@ -6234,10 +5726,6 @@ packages: darkreader@4.9.95: resolution: {integrity: sha512-P3sRqPsOcEs8k/36BEBhVrdY1nYYF03kK6vfQ7oLBzwuCpSanambl6xxsdoW/fyKevSRriBkU4LRmQrUxPIRew==} - data-uri-to-buffer@4.0.1: - resolution: {integrity: sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==} - engines: {node: '>= 12'} - data-uri-to-buffer@6.0.2: resolution: {integrity: sha512-7hvf7/GW8e86rW0ptuwS3OcBGDjIi6SZva7hCyWC0yYry2cOPmLIjXAUHI6DK2HsnwJd9ifmt57i8eV2n4YNpw==} engines: {node: '>= 14'} @@ -6280,6 +5768,15 @@ packages: supports-color: optional: true + debug@4.3.7: + resolution: {integrity: sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + debug@4.4.1: resolution: {integrity: sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==} engines: {node: '>=6.0'} @@ -6297,10 +5794,6 @@ packages: resolution: {integrity: sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA==} engines: {node: '>=0.10.0'} - decamelize@4.0.0: - resolution: {integrity: sha512-9iE1PgSik9HeIIw2JO94IidnE3eBoQrFJ3w7sFuzSX4DpmZ3v5sZpUiV5Swcf6mQEF+Y0ru8Neo+p+nyh2J+hQ==} - engines: {node: '>=10'} - decimal.js-light@2.5.1: resolution: {integrity: sha512-qIMFpTMZmny+MMIitAB6D7iVPEorVw6YQRWkvarTkT4tBeSLLiHzcwj6q0MmYSFCiVpiqPJTJEYIrpcPzVEIvg==} @@ -6322,10 +5815,6 @@ packages: babel-plugin-macros: optional: true - deep-equal@1.1.2: - resolution: {integrity: sha512-5tdhKF6DbU7iIzrIOa1AOUt39ZRm13cmL1cGEh//aqR8x9+tNfbywRf0n5FD/18OKMdo7DNEtrX2t22ZAkI+eg==} - engines: {node: '>= 0.4'} - deep-extend@0.6.0: resolution: {integrity: sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==} engines: {node: '>=4.0.0'} @@ -6378,6 +5867,10 @@ packages: resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} engines: {node: '>=0.4.0'} + denque@2.1.0: + resolution: {integrity: sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==} + engines: {node: '>=0.10'} + depd@1.1.2: resolution: {integrity: sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==} engines: {node: '>= 0.6'} @@ -6413,6 +5906,9 @@ packages: detect-node@2.1.0: resolution: {integrity: sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g==} + devlop@1.1.0: + resolution: {integrity: sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA==} + dezalgo@1.0.4: resolution: {integrity: sha512-rXSP0bf+5n0Qonsb+SVVfNfIsimO4HEtmnIpPHY8Q1UCzKlQrDMfdobr8nJOOsRgWCyMRqeSBQzmWUMq7zvVig==} @@ -6430,14 +5926,6 @@ packages: resolution: {integrity: sha512-EjePK1srD3P08o2j4f0ExnylqRs5B9tJjcp9t1krH2qRi8CCdsYfwe9JgSLurFBWwq4uOlipzfk5fHNvwFKr8Q==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} - diff@3.5.0: - resolution: {integrity: sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==} - engines: {node: '>=0.3.1'} - - diff@5.2.0: - resolution: {integrity: sha512-uIFDxqpRZGZ6ThOk84hEfqWoHx2devRFvpTZcTHur85vImfaxUbTW9Ryh4CpCuDnToOP1CEtXKIgytHBPVff5A==} - engines: {node: '>=0.3.1'} - dir-glob@3.0.1: resolution: {integrity: sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==} engines: {node: '>=8'} @@ -6446,6 +5934,9 @@ packages: resolution: {integrity: sha512-GYqKi1aH7PJXxdhTeZBFrg8vUBeKXi+cNprXsC1kpJcbcVnV9wBsrOu1cQEdG0WeQwlfHiy3XvnKfIrJ2R0NzQ==} hasBin: true + discontinuous-range@1.0.0: + resolution: {integrity: sha512-c68LpLbO+7kP/b1Hr1qs8/BJ09F5khZGTxqxZuhzxpmwJKOgRFHJWIb9/KmqnqHhLdO55aOxFH/EGBvUQbL/RQ==} + diskusage@1.2.0: resolution: {integrity: sha512-2u3OG3xuf5MFyzc4MctNRUKjjwK+UkovRYdD2ed/NZNZPrt0lqHnLKxGhlFVvAb4/oufIgQG3nWgwmeTbHOvXA==} @@ -6567,8 +6058,8 @@ packages: engines: {node: '>=0.10.0'} hasBin: true - electron-to-chromium@1.5.136: - resolution: {integrity: sha512-kL4+wUTD7RSA5FHx5YwWtjDnEEkIIikFgWHR4P6fqjw1PPLlqYkxeOb++wAauAssat0YClCy8Y3C5SxgSkjibQ==} + electron-to-chromium@1.5.157: + resolution: {integrity: sha512-/0ybgsQd1muo8QlnuTpKwtl0oX5YMlUGbm8xyqgDU00motRkKFFbUJySAQBWcY79rVqNLWIWa87BGVGClwAB2w==} electron-to-chromium@1.5.167: resolution: {integrity: sha512-LxcRvnYO5ez2bMOFpbuuVuAI5QNeY1ncVytE/KXaL6ZNfzX1yPlAO0nSOyIHx2fVAuUprMqPs/TdVhUFZy7SIQ==} @@ -6606,11 +6097,6 @@ packages: enabled@2.0.0: resolution: {integrity: sha512-AKrN98kuwOzMIdAizXGI86UFBoo26CL21UM763y1h/GMSJ4/OHU9k2YlsmBpyScFo/wbLzWQJBMCW4+IO3/+OQ==} - enchannel-zmq-backend@9.1.23: - resolution: {integrity: sha512-yBSCr2DZcKqTZhJME5jvAiU0waMiEHU8iGxeezbEqQAVQUZR9IZyKLU8Q6RZMIIUk2alzeeh2WVuYRWPrJXtpQ==} - peerDependencies: - rxjs: ^6.3.3 - encodeurl@1.0.2: resolution: {integrity: sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==} engines: {node: '>= 0.8'} @@ -6625,6 +6111,17 @@ packages: end-of-stream@1.4.4: resolution: {integrity: sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==} + engine.io-client@6.6.3: + resolution: {integrity: sha512-T0iLjnyNWahNyv/lcjS2y4oE358tVS/SYQNxYXGAJ9/GLgH4VCvOQ/mhTjqU88mLZCQgiG8RIegFHYCdVC+j5w==} + + engine.io-parser@5.2.3: + resolution: {integrity: sha512-HqD3yTBfnBxIrbnM1DoD6Pcq8NECnh8d4As1Qgh0z5Gg3jRRIqijury0CL3ghu/edArpUYiYqQiDUQBIs4np3Q==} + engines: {node: '>=10.0.0'} + + engine.io@6.6.4: + resolution: {integrity: sha512-ZCkIjSYNDyGn0R6ewHDtXgns/Zre/NT6Agvq1/WobF7JXgFff4SeDroKiCO3fNJreU9YG429Sc81o4w5ok/W5g==} + engines: {node: '>=10.2.0'} + enhanced-resolve@5.18.1: resolution: {integrity: sha512-ZSW3ma5GkcQBIpwZTSRAI8N71Uuwgs93IezB7mf7R60tC8ZbJideoDNKjHn2O9KIlx6rkGTTEk1xUCK2E1Y2Yg==} engines: {node: '>=10.13.0'} @@ -6651,6 +6148,12 @@ packages: env-variable@0.0.6: resolution: {integrity: sha512-bHz59NlBbtS0NhftmR8+ExBEekE7br0e01jw+kk0NDro7TtZzBYZ5ScGPs3OmwnpyfHTHOtr1Y6uedCdrIldtg==} + enzyme-shallow-equal@1.0.7: + resolution: {integrity: sha512-/um0GFqUXnpM9SvKtje+9Tjoz3f1fpBC3eXRFrNs8kpYn69JljciYP7KZTqM/YQbUY9KUjvKB4jo/q+L6WGGvg==} + + enzyme@3.11.0: + resolution: {integrity: sha512-Dw8/Gs4vRjxY6/6i9wU0V+utmQO9kvh9XLnz3LIudviOnVYDEe2ec+0k+NQoMamn1VrjKgCUOWj5jG/5M5M0Qw==} + errno@0.1.8: resolution: {integrity: sha512-dJ6oBr5SQ1VSd9qkk7ByRgb/1SH4JZjCHSW/mr63/QcXO9zLVxvJ6Oy13nio03rxpSnVDDjFor75SjVeZWPW/A==} hasBin: true @@ -6665,6 +6168,9 @@ packages: resolution: {integrity: sha512-py07lI0wjxAC/DcfK1S6G7iANonniZwTISvdPzk9hzeH0IZIshbuuFxLIU96OyF89Yb9hiqWn8M/bY83KY5vzA==} engines: {node: '>= 0.4'} + es-array-method-boxes-properly@1.0.0: + resolution: {integrity: sha512-wd6JXUmyHmt8T5a2xreUwKcGPq6f1f+WwIJkijUqiGcJz1qqnZgP6XIK+QyIWU5lT7imeNxUll48bziG+TSYcA==} + es-class@2.1.1: resolution: {integrity: sha512-loFNtCIGY81XvaHMzsxPocOgwZW71p+d/iES+zDSWeK9D4JaxrR/AoO0sZnWbV39D/ESppKbHrApxMi+Vbl8rg==} @@ -6862,10 +6368,6 @@ packages: resolution: {integrity: sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==} engines: {node: '>=10'} - execa@8.0.1: - resolution: {integrity: sha512-VyhnebXciFV2DESc+p6B+y0LjSm0krU4OgJN44qFAhBY0TJ+1V61tYD2+wHusZ6F9n5K+vl8k0sTy7PEfV4qpg==} - engines: {node: '>=16.17'} - exit-hook@4.0.0: resolution: {integrity: sha512-Fqs7ChZm72y40wKjOFXBKg7nJZvQJmewP5/7LtePDdnah/+FH9Hp5sgMujSCMPXlxOAW2//1jrW9pnsY7o20vQ==} engines: {node: '>=18'} @@ -6934,9 +6436,6 @@ packages: resolution: {integrity: sha512-V7/RktU11J3I36Nwq2JnZEM7tNm17eBJz+u25qdxBZeCKiX6BkVSZQjwWIr+IobgnZy+ag73tTZgZi7tr0LrBw==} engines: {node: '>=6.0.0'} - fast-fifo@1.3.2: - resolution: {integrity: sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==} - fast-glob@3.3.2: resolution: {integrity: sha512-oX2ruAFQwf/Orj8m737Y5adxDQO0LAB7/S5MnxCdTNDd4p6BsyIVsv9JQsATbTSq8KHRpLwIHbVlUNatxd+1Ow==} engines: {node: '>=8.6.0'} @@ -6977,21 +6476,14 @@ packages: fb-watchman@2.0.2: resolution: {integrity: sha512-p5161BqbuCaSnB8jIbzQHOlpgsPmK5rJVDfDKO91Axs5NC1uu3HRQm6wt9cd9/+GtQQIO53JdGXXoyDpTAsgYA==} - fdir@6.4.6: - resolution: {integrity: sha512-hiFoqpyZcfNm1yc4u8oWCf9A2c4D3QjCrks3zmoVKVxpQRzmPNar1hUJcBG2RQHvEVGDN+Jm81ZheVLAQMK6+w==} + fdir@6.4.4: + resolution: {integrity: sha512-1NZP+GK4GfuAv3PqKvxQRDMjdSRZjnkq7KfhlNrCNNlZ0ygQFpebfrnfnq/W7fpUnAv9aGWmY1zKx7FYL3gwhg==} peerDependencies: picomatch: ^3 || ^4 peerDependenciesMeta: picomatch: optional: true - fetch-blob@3.2.0: - resolution: {integrity: sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==} - engines: {node: ^12.20 || >= 14.13} - - fflate@0.7.3: - resolution: {integrity: sha512-0Zz1jOzJWERhyhsimS54VTqOteCNwRtIlh8isdL0AXLo0g7xNTfTL7oWrkmCnPhZGocKIkWHBistBrrpoNH3aw==} - fflate@0.8.2: resolution: {integrity: sha512-cPJU47OaAoCbg0pBvzsgpTPhmhqI5eJjh/JIu8tPj5q+T7iLvW/JAYUqmE7KOB4R1ZyEhzBaIQpQpardBF5z8A==} @@ -7055,10 +6547,6 @@ packages: flat-zip@1.0.1: resolution: {integrity: sha512-s/8bbMuRP3YOBYlpcUzmOiJelXpzSGogbZrXtdHUtoO6O0gEcfOCDDkivJ+9zOwNgzgPQOpJX1v6YwBfPQiYqQ==} - flat@5.0.2: - resolution: {integrity: sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ==} - hasBin: true - flatted@3.3.3: resolution: {integrity: sha512-GX+ysw4PBCz0PzosHDepZGANEuFCMLrnRTiEy9McGjmkCQYwRq4A/X786G/fjM/+OjsWSU1ZrY5qyARZmO/uwg==} @@ -7103,10 +6591,6 @@ packages: resolution: {integrity: sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==} engines: {node: '>= 12.20'} - formdata-polyfill@4.0.10: - resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==} - engines: {node: '>=12.20.0'} - formidable@3.5.2: resolution: {integrity: sha512-Jqc1btCy3QzRbJaICGwKcBfGWuLADRerLzDqi2NwSt/UkXLsHJw2TVResiaoBufHVHy9aSgClOHCeJsSsFLTbg==} @@ -7165,18 +6649,10 @@ packages: resolution: {integrity: sha512-LDODD4TMYx7XXdpwxAVRAIAuB0bzv0s+ywFonY46k126qzQHT9ygyoa9tncmOiQmmDrik65UYsEkv3lbfqQ3yQ==} engines: {node: '>=14'} - gaxios@7.0.0-rc.4: - resolution: {integrity: sha512-fwQMwbs3o8Odl/nc/rkQJwyHeOXdderOwmybUl0gkyTdZXMK1oSTWj4Em7gSogVJsRWDeHPXLY06+e8Rkr01iw==} - engines: {node: '>=18'} - gcp-metadata@6.1.1: resolution: {integrity: sha512-a4tiq7E0/5fTjxPAaH4jpjkSv/uCaU2p5KC6HVGrvl0cDjA8iBZv4vv1gyzlmK0ZUKqwpOyQMKzZQe3lTit77A==} engines: {node: '>=14'} - gcp-metadata@7.0.0-rc.1: - resolution: {integrity: sha512-E6c+AdIaK1LNA839OyotiTca+B2IG1nDlMjnlcck8JjXn3fVgx57Ib9i6iL1/iqN7bA3EUQdcRRu+HqOCOABIg==} - engines: {node: '>=18'} - gensequence@7.0.0: resolution: {integrity: sha512-47Frx13aZh01afHJTB3zTtKIlFI6vWY+MYCN9Qpew6i52rfKjnhCF/l1YlC8UmEMvvntZZ6z4PiCcmyuedR2aQ==} engines: {node: '>=18'} @@ -7234,10 +6710,6 @@ packages: resolution: {integrity: sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==} engines: {node: '>=10'} - get-stream@8.0.1: - resolution: {integrity: sha512-VaUJspBffn/LMCJVoMvSAdmscJyS1auj5Zulnn5UoYcY531UWmdwhRWkcGKnGU93m5HSXP9LP2usOryrBtQowA==} - engines: {node: '>=16'} - get-symbol-description@1.1.0: resolution: {integrity: sha512-w9UMqWwJxHNOvoNzSJ2oPF5wvYcvP7jUvYzhp67yEhTi17ZDBBC1z9pTdGuzjD+EFIqLSYRweZjqfiPzQ06Ebg==} engines: {node: '>= 0.4'} @@ -7284,11 +6756,6 @@ packages: resolution: {integrity: sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==} deprecated: Glob versions prior to v9 are no longer supported - glob@8.1.0: - resolution: {integrity: sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==} - engines: {node: '>=12'} - deprecated: Glob versions prior to v9 are no longer supported - glob@9.3.5: resolution: {integrity: sha512-e1LleDykUz2Iu+MTYdkSsuWX8lvAjAcs0Xef0lNIu0S2wOAzuTxCJtcd9S3cijlwYF18EsU3rzb8jPVobxDh9Q==} engines: {node: '>=16 || 14 >=14.17'} @@ -7378,13 +6845,6 @@ packages: resolution: {integrity: sha512-bud98CJ6kGZcP9Yxcsi7Iz647wuDz3oN+IZsjCRi5X1PI7t/xPKeL0mOwXJjo+CRZMqvq0CkSJiywCcY7kVYog==} hasBin: true - glur@1.1.2: - resolution: {integrity: sha512-l+8esYHTKOx2G/Aao4lEQ0bnHWg4fWtJbVoZZT9Knxi01pB8C80BR85nONLFwkkQoFRCmXY+BUcGZN3yZ2QsRA==} - - google-auth-library@10.0.0-rc.1: - resolution: {integrity: sha512-Ri8Yk7bMhaPcqzwyW5XHS4scc5KL+AdyUVxA5YGw9BUxYcL2P/tdEfj13O9KpV03k5sUqlaTL+HPxb/deGVjxw==} - engines: {node: '>=18'} - google-auth-library@9.15.1: resolution: {integrity: sha512-Jb6Z0+nvECVz+2lzSMt9u98UsoakXxA2HGHMCxh+so3n90XgYWkq5dur19JAJV7ONiJY22yBTyJB1TSkvPq9Ng==} engines: {node: '>=14'} @@ -7393,18 +6853,10 @@ packages: resolution: {integrity: sha512-Phyp9fMfA00J3sZbJxbbB4jC55b7DBjE3F6poyL3wKMEBVKA79q6BGuHcTiM28yOzVql0NDbRL8MLLh8Iwk9Dg==} engines: {node: '>=14'} - google-gax@5.0.1-rc.0: - resolution: {integrity: sha512-7dfJz21VjUYfVB8XKfXw2ETvjUp1Oa2DUFiNqnJ2k+AY0BDDeqIe4H5ROmLw8dRfv+TwmCL4DJELPRuOh8XAGA==} - engines: {node: '>=18'} - google-logging-utils@0.0.2: resolution: {integrity: sha512-NEgUnEcBiP5HrPzufUkBzJOD/Sxsco3rLNo1F1TNf7ieU8ryUzBhqba8r756CjLX7rn3fHl6iLEwPYuqpoKgQQ==} engines: {node: '>=14'} - google-logging-utils@1.1.1: - resolution: {integrity: sha512-rcX58I7nqpu4mbKztFeOAObbomBbHU2oIb/d3tJfF3dizGSApqtSwYJigGCooHdnMyQBIw8BrWyK96w3YXgr6A==} - engines: {node: '>=14'} - googleapis-common@7.2.0: resolution: {integrity: sha512-/fhDZEJZvOV3X5jmD+fKxMqma5q2Q9nZNSF3kn1F18tpxmA86BcTxAGBQdM0N89Z3bEaIs+HVznSmFJEAmMTjA==} engines: {node: '>=14.0.0'} @@ -7413,10 +6865,6 @@ packages: resolution: {integrity: sha512-2L7SzN0FLHyQtFmyIxrcXhgust77067pkkduqkbIpDuj9JzVnByxsRrcRfUMFQam3rQkWW2B0f1i40IwKDWIVQ==} engines: {node: '>=14.0.0'} - googlediff@0.1.0: - resolution: {integrity: sha512-71ZD3jCKckWRnh1DTPJABge1uZwwqMTUD1qyiao2VkAKvsO1keNf7/PGKXaDNGrq7EHOm1Z8TVT5jvG3SQuZAg==} - engines: {node: '>=0.8.0'} - gopd@1.2.0: resolution: {integrity: sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==} engines: {node: '>= 0.4'} @@ -7442,10 +6890,6 @@ packages: resolution: {integrity: sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw==} engines: {node: '>=14.0.0'} - gtoken@8.0.0-rc.1: - resolution: {integrity: sha512-UjE/egX6ixArdcCKOkheuFQ4XN4/0gX92nd2JPVEYuRU2sWHAWuOVGnowm1fQUdQtaxqn1n8H0hOb2LCaUhJ3A==} - engines: {node: '>=18'} - gzip-size@6.0.0: resolution: {integrity: sha512-ax7ZYomf6jqPTQ4+XCpUGyXKHk5WweS+e05MBO4/y3WJ5RkmPXNKvX+bx1behVILVwr6JSQvZAku021CHPXG3Q==} engines: {node: '>=10'} @@ -7514,34 +6958,41 @@ packages: resolution: {integrity: sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==} engines: {node: '>= 0.4'} + has@1.0.4: + resolution: {integrity: sha512-qdSAmqLF6209RFj4VVItywPMbm3vWylknmB3nvNiUIs72xAimcM8nVYxYr7ncvZq5qzk9MKIZR8ijqD/1QuYjQ==} + engines: {node: '>= 0.4.0'} + hasown@2.0.2: resolution: {integrity: sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==} engines: {node: '>= 0.4'} - hast-util-from-parse5@7.1.2: - resolution: {integrity: sha512-Nz7FfPBuljzsN3tCQ4kCBKqdNhQE2l0Tn+X1ubgKBPRoiDIu1mL08Cfw4k7q71+Duyaw7DXDN+VTAp4Vh3oCOw==} + hast-util-from-html@2.0.3: + resolution: {integrity: sha512-CUSRHXyKjzHov8yKsQjGOElXy/3EKpyX56ELnkHH34vDVw1N1XSQ1ZcAvTyAPtGqLTuKP/uxM+aLkSPqF/EtMw==} + + hast-util-from-parse5@8.0.3: + resolution: {integrity: sha512-3kxEVkEKt0zvcZ3hCRYI8rqrgwtlIOFMWkbclACvjlDw8Li9S2hk/d51OI0nr/gIpdMHNepwgOKqZ/sy0Clpyg==} hast-util-parse-selector@3.1.1: resolution: {integrity: sha512-jdlwBjEexy1oGz0aJ2f4GKMaVKkA9jwjr4MjAAI22E5fM/TXVZHuS5OpONtdeIkRKqAaryQ2E9xNQxijoThSZA==} - hast-util-raw@7.2.3: - resolution: {integrity: sha512-RujVQfVsOrxzPOPSzZFiwofMArbQke6DJjnFfceiEbFh7S05CbPt0cYN+A5YeD3pso0JQk6O1aHBnx9+Pm2uqg==} + hast-util-parse-selector@4.0.0: + resolution: {integrity: sha512-wkQCkSYoOGCRKERFWcxMVMOcYE2K1AaNLU8DXS9arxnLOUEWbOXKXiJUNzEpqZ3JOKpnha3jkFrumEjVliDe7A==} - hast-util-to-html@8.0.4: - resolution: {integrity: sha512-4tpQTUOr9BMjtYyNlt0P50mH7xj0Ks2xpo8M943Vykljf99HW6EzulIoJP1N3eKOSScEHzyzi9dm7/cn0RfGwA==} + hast-util-to-html@9.0.5: + resolution: {integrity: sha512-OguPdidb+fbHQSU4Q4ZiLKnzWo8Wwsf5bZfbvu7//a9oTYoqD/fWpe96NuHkoS9h0ccGOTe0C4NGXdtS0iObOw==} - hast-util-to-parse5@7.1.0: - resolution: {integrity: sha512-YNRgAJkH2Jky5ySkIqFXTQiaqcAtJyVE+D5lkN6CdtOqrnkLfGYYrEcKuHOJZlp+MwjSwuD3fZuawI+sic/RBw==} + hast-util-to-string@3.0.1: + resolution: {integrity: sha512-XelQVTDWvqcl3axRfI0xSeoVKzyIFPwsAGSLIsKdJKQMXDYJS4WYrBNF/8J7RdhIcFI2BOHgAifggsvsxp/3+A==} - hast-util-to-string@2.0.0: - resolution: {integrity: sha512-02AQ3vLhuH3FisaMM+i/9sm4OXGSq1UhOOCpTLLQtHdL3tZt7qil69r8M8iDkZYyC0HCFylcYoP+8IO7ddta1A==} - - hast-util-whitespace@2.0.1: - resolution: {integrity: sha512-nAxA0v8+vXSBDt3AnRUNjyRIQ0rD+ntpbAp4LnPkumc5M9yUbSMa4XDU9Q6etY4f1Wp4bNgvc1yjiZtsTTrSng==} + hast-util-whitespace@3.0.0: + resolution: {integrity: sha512-88JUN06ipLwsnv+dVn+OIYOvAuvBMy/Qoi6O7mQHxdPXpjy+Cd6xRkWwux7DKO+4sYILtLBRIKgsdpS2gQc7qw==} hastscript@7.2.0: resolution: {integrity: sha512-TtYPq24IldU8iKoJQqvZOuhi5CyCQRAbvDOX0x1eW6rsHSxa/1i2CCiptNTotGHJ3VoHRGmqiv6/D3q113ikkw==} + hastscript@9.0.1: + resolution: {integrity: sha512-g7df9rMFX/SPi34tyGCyUBREQoKkapwdY/T04Qn9TDWfHhAYt4/I0gMVirzK5wEzeUqIjEB+LXC/ypb7Aqno5w==} + he@1.2.0: resolution: {integrity: sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==} hasBin: true @@ -7553,9 +7004,6 @@ packages: highlight-words-core@1.2.3: resolution: {integrity: sha512-m1O9HW3/GNHxzSIXWw1wCNXXsgLlxrP0OI6+ycGUhiUHkikqW3OrwVHz+lxeNBe5yqLESdIcj8PowHQ2zLvUvQ==} - history@1.17.0: - resolution: {integrity: sha512-hLthvd5fW7WlZR+hb2iqaVJLskcUAnQUrSnvBQCLdVmtf0zdAymoBgSp7v05EULc/jIn0qwPUf2BQBhKN1YeGQ==} - hoist-non-react-statics@3.3.2: resolution: {integrity: sha512-/gGivxi8JPKWNm/W0jSmzcMPpfpPLc3dY/6GxhX2hQ9iGj3aDfklV4ET7NjKpSinLpJ5vafa9iiGIEZg10SfBw==} @@ -7578,6 +7026,9 @@ packages: html-dom-parser@1.2.0: resolution: {integrity: sha512-2HIpFMvvffsXHFUFjso0M9LqM+1Lm22BF+Df2ba+7QHJXjk63pWChEnI6YG27eaWqUdfnh5/Vy+OXrNTtepRsg==} + html-element-map@1.3.1: + resolution: {integrity: sha512-6XMlxrAFX4UEEGxctfFnmrFaaZFNf9i5fNuV5wZ3WWQ4FVaNP1aX1LkX9j2mfEx1NpjeE/rL3nmgEn23GdFmrg==} + html-entities@2.6.0: resolution: {integrity: sha512-kig+rMn/QOVRvr7c86gQ8lWXq+Hkv6CbAH1hLu+RG338StTpE8Z0b44SDVaqVu7HGKf27frdmUYEs9hTUX/cLQ==} @@ -7605,8 +7056,8 @@ packages: peerDependencies: react: 0.14 || 15 || 16 || 17 || 18 - html-void-elements@2.0.1: - resolution: {integrity: sha512-0quDb7s97CfemeJAnW9wC0hw78MtW7NU3hqtCD75g2vFlDLt36llsYD7uB7SUzojLMP24N5IatXf7ylGXiGG9A==} + html-void-elements@3.0.0: + resolution: {integrity: sha512-bEqo66MRXsUGxWHV5IP0PUiAWwoEjba4VCzg0LjFJBpchPaTfyfCKTG6bc5F8ucKec3q5y6qOdGyYTSBEvhCrg==} html-webpack-plugin@5.6.3: resolution: {integrity: sha512-QSf1yjtSAsmf7rYBV7XX86uua4W/vkhIt0xNXKbsi2foEeW7vjJQz4bhnpL3xH+l1ryl1680uNv968Z+X6jSYg==} @@ -7629,10 +7080,6 @@ packages: htmlparser2@8.0.2: resolution: {integrity: sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==} - htmlparser@1.7.7: - resolution: {integrity: sha512-zpK66ifkT0fauyFh2Mulrq4AqGTucxGtOhZ8OjkbSfcCpkqQEI8qRkY0tSQSJNAQ4HUZkgWaU4fK4EH6SVH9PQ==} - engines: {node: '>=0.1.33'} - http-deceiver@1.2.7: resolution: {integrity: sha512-LmpOGxTfbpgtGVxJrj5k7asXHCgNZp5nLfp+hWc8QQRqtb7fUy6kRY3BO1h9ddF6yIPYUARgxGOwB42DnxIaNw==} @@ -7684,10 +7131,6 @@ packages: resolution: {integrity: sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==} engines: {node: '>=10.17.0'} - human-signals@5.0.0: - resolution: {integrity: sha512-AXcZb6vzzrFAUE61HnN4mpLqd/cSIwNQjtNWR0euPm6y0iqx3G4gOXaIDdtdDwZmhwe82LA6+zinmW4UBWVePQ==} - engines: {node: '>=16.17.0'} - humanize-list@1.0.1: resolution: {integrity: sha512-4+p3fCRF21oUqxhK0yZ6yaSP/H5/wZumc7q1fH99RkW7Q13aAxDeP78BKjoR+6y+kaHqKF/JWuQhsNuuI2NKtA==} @@ -7795,10 +7238,6 @@ packages: resolution: {integrity: sha512-M1WuAmb7pn9zdFRtQYk26ZBoY043Sse0wVDdk4Bppr+JOXyQYybdtvK+l9wUibhtjdjvtoiNy8tk+EgsYIUqKg==} engines: {node: '>=12.0.0'} - install@0.13.0: - resolution: {integrity: sha512-zDml/jzr2PKU9I8J/xyZBQn8rPCAY//UOYNmR01XwNwyfhEWObo2SWfSl1+0tm1u6PhxLwDnfsT/6jB7OUxqFA==} - engines: {node: '>= 0.10'} - internal-slot@1.1.0: resolution: {integrity: sha512-4gd7VpWNQNB4UKKCFFVcp1AVv+FMOgs9NKzjHKusc8jTMhd5eL1NqQqOpE0KzMds804/yHlglp3uxgluOqAPLw==} engines: {node: '>= 0.4'} @@ -7817,8 +7256,9 @@ packages: intl-messageformat@10.7.16: resolution: {integrity: sha512-UmdmHUmp5CIKKjSoE10la5yfU+AYJAaiYLsodbjL4lji83JNvgOQUjGaGhGrpFCb0Uh7sl7qfP1IyILa8Z40ug==} - invariant@2.2.4: - resolution: {integrity: sha512-phJfQVBuaJM5raOpJjSfkiD6BpbCE4Ns//LaXl6wGYtUBY83nWS6Rf9tXm2e8VaK60JEjYldbPif/A2B1C2gNA==} + iovalkey@0.3.3: + resolution: {integrity: sha512-4rTJX6Q5wTYEvxboXi8DsEiUo+OvqJGtLYOSGm37KpdRXsG5XJjbVtYKGJpPSWP+QT7rWscA4vsrdmzbEbenpw==} + engines: {node: '>=18.12.0'} ip-address@9.0.5: resolution: {integrity: sha512-zHtQzGojZXTwZTHQqra+ETKd4Sn3vgi7uBmlPoXVWZqYvuKmtI0l/VZTjqGmJY9x88GGOaZ9+G9ES8hC4T4X8g==} @@ -7882,10 +7322,6 @@ packages: is-buffer@1.1.6: resolution: {integrity: sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==} - is-buffer@2.0.5: - resolution: {integrity: sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ==} - engines: {node: '>=4'} - is-callable@1.2.7: resolution: {integrity: sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA==} engines: {node: '>= 0.4'} @@ -7894,6 +7330,10 @@ packages: resolution: {integrity: sha512-z0vtXSwucUJtANQWldhbtbt7BnL0vxiFjIdDLAatwhDYty2bad6s+rijD6Ri4YuYJubLzIJLUidCh09e1djEVQ==} engines: {node: '>= 0.4'} + is-core-module@2.16.1: + resolution: {integrity: sha512-UfoeMA6fIJ8wTYFEUjelnaGI67v6+N7qXJEvQuIGa99l4xsCruSYOVSQ0uPANn4dAzm8lkYPaKLrrijLq7x23w==} + engines: {node: '>= 0.4'} + is-data-view@1.0.2: resolution: {integrity: sha512-RKtWF8pGmS87i2D6gqQu/l7EYRlVdfzemCJN/P3UOs//x1QE7mfhvzHIApBTRf7axvT6DMGwSwBXYCT0nfB9xw==} engines: {node: '>= 0.4'} @@ -8064,10 +7504,6 @@ packages: resolution: {integrity: sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==} engines: {node: '>=8'} - is-stream@3.0.0: - resolution: {integrity: sha512-LnQR4bZ9IADDRSkvpqMGvt/tEJWclzklNgSw48V5EAaAeDd6qGvN8ei6k5p0tvxSR171VmGyHuTiAOfxAbr8kA==} - engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} - is-string-blank@1.0.1: resolution: {integrity: sha512-9H+ZBCVs3L9OYqv8nuUAzpcT9OTgMD1yAWrG7ihlnibdkbtB850heAmYWxHuXc4CHy4lKeK69tN+ny1K7gBIrw==} @@ -8075,6 +7511,9 @@ packages: resolution: {integrity: sha512-BtEeSsoaQjlSPBemMQIrY1MY0uM6vnS1g5fmufYOtnxLGUZM2178PKbhsk7Ffv58IX+ZtcvoGwccYsh0PglkAA==} engines: {node: '>= 0.4'} + is-subset@0.1.1: + resolution: {integrity: sha512-6Ybun0IkarhmEqxXCNw/C0bna6Zb/TkfUX9UbwJtK6ObwAVCxmAP308WWTHviM/zAqXk05cdhYsUsZeGQh99iw==} + is-svg-path@1.0.2: resolution: {integrity: sha512-Lj4vePmqpPR1ZnRctHv8ltSh1OrSxHkhUkd7wi+VQdcdP15/KvQFyk7LhNuM7ZW0EVbJz8kZLVmL9quLrfq4Kg==} @@ -8331,10 +7770,6 @@ packages: node-notifier: optional: true - jmp@2.0.0: - resolution: {integrity: sha512-VATfWVHErQJA2XMtmQjJQHHyQ/hxjHMmsy+egmwRk/RzFchQB4xjrR1iX496VZr+Hyhcr4zvL+IkkSlIYKx6Yw==} - engines: {node: '>=6'} - jquery-focus-exit@1.0.1: resolution: {integrity: sha512-p79pTJKVsrYX4bZML+bc1MnRe9IrxcpIIeUqyfuVMpLFR4L05r7zxBe3YywPjn74hNavuOA2KNBhy36UXroMiA==} peerDependencies: @@ -8377,9 +7812,6 @@ packages: jquery-ui@1.14.1: resolution: {integrity: sha512-DhzsYH8VeIvOaxwi+B/2BCsFFT5EGjShdzOcm5DssWjtcpGWIMsn66rJciDA6jBruzNiLf1q0KvwMoX1uGNvnQ==} - jquery.payment@3.0.0: - resolution: {integrity: sha512-b02U7bf5HIhmfwsODomtLe119VeZ4c5hBvyKkdV3Ih4tTrD08sEPZuhgrX0Sv9iWIb08vNWn4RpaCyZ48mhOJA==} - jquery.scrollintoview@1.9.4: resolution: {integrity: sha512-mAeLHnu6HA01W4BWWZgovLE5Gr8qgs6BGCaMB5oha3YfAKZ2mJhAqd1CbDZsFnXMvgxQj70JYXzyftolQLAwFg==} @@ -8492,9 +7924,6 @@ packages: jupyter-paths@2.0.4: resolution: {integrity: sha512-hS2osbroOqfcEsnkTHNkdDhNs0dwhR7/k57msC0iLo03pb6dqVMjpiBMxHhJ4/XNddhoc1SU3SdAk+pg2VuiRw==} - just-extend@4.2.1: - resolution: {integrity: sha512-g3UB796vUFIY90VIv/WX3L2c8CS2MdWUww3CNrYmqza1Fg0DURc2K/O4YrnklBdQarSJ/y8JnJYDGc+1iumQjg==} - jwa@1.4.1: resolution: {integrity: sha512-qiLX/xhEEFKUAJ6FiBMbes3w9ATzyk5W7Hvzpa/SLYdxNtng+gcurvrI7TbACjIXlsJyr05/S1oUhZrc63evQA==} @@ -8561,8 +7990,8 @@ packages: langs@2.0.0: resolution: {integrity: sha512-v4pxOBEQVN1WBTfB1crhTtxzNLZU9HPWgadlwzWKISJtt6Ku/CnpBrwVy+jFv8StjxsPfwPFzO0CMwdZLJ0/BA==} - langsmith@0.3.31: - resolution: {integrity: sha512-9lwuLZuN3tXFYQ6eMg0rmbBw7oxQo4bu1NYeylbjz27bOdG1XB9XNoxaiIArkK4ciLdOIOhPMBXP4bkvZOgHRw==} + langsmith@0.3.29: + resolution: {integrity: sha512-JPF2B339qpYy9FyuY4Yz1aWYtgPlFc/a+VTj3L/JcFLHCiMP7+Ig8I9jO+o1QwVa+JU3iugL1RS0wwc+Glw0zA==} peerDependencies: openai: '*' peerDependenciesMeta: @@ -8582,21 +8011,11 @@ packages: resolution: {integrity: sha512-/tFkx5WIn4HuO+6w9lsfxq4FN3O+fDZeO9Mek8dCD8rTUpqzRa766BOBO7BcGkn3X86m5+cBm1/2S/Shzz7gMg==} engines: {node: '>=0.8'} - ldapauth-fork@5.0.5: - resolution: {integrity: sha512-LWUk76+V4AOZbny/3HIPQtGPWZyA3SW2tRhsWIBi9imP22WJktKLHV1ofd8Jo/wY7Ve6vAT7FCI5mEn3blZTjw==} - engines: {node: '>=0.8.0'} - ldapjs@2.3.3: resolution: {integrity: sha512-75QiiLJV/PQqtpH+HGls44dXweviFwQ6SiIK27EqzKQ5jU/7UFrl2E5nLdQ3IYRBzJ/AVFJI66u0MZ0uofKYwg==} engines: {node: '>=10.13.0'} deprecated: This package has been decomissioned. See https://github.com/ldapjs/node-ldapjs/blob/8ffd0bc9c149088a10ec4c1ec6a18450f76ad05d/README.md - lean-client-js-core@1.5.0: - resolution: {integrity: sha512-1uQ+ldW85sMW1zvwab424qGM1bDwJ2d51qcUt5bxWPIp9Zef7pcqRhYytAvxj1G1+4+gZhUzbyHS7wfh7bRrcg==} - - lean-client-js-node@1.5.0: - resolution: {integrity: sha512-1Vx4huU6FCN1ZNNbTghXlJKQH1RoxhNnQCTxRkeFRUkDyM5p7BYn8XuCMacwuiDH21y1NEP7BNbiyNBXCULMWA==} - less-loader@11.1.4: resolution: {integrity: sha512-6/GrYaB6QcW6Vj+/9ZPgKKs6G10YZai/l/eJ4SLwbzqNTBsAqt5hSLVF47TgsiBxV1P6eAU0GYRH3YRuQU9V3A==} engines: {node: '>= 14.15.0'} @@ -8687,16 +8106,28 @@ packages: lodash.debounce@4.0.8: resolution: {integrity: sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow==} - lodash.get@4.4.2: - resolution: {integrity: sha512-z+Uw/vLuy6gQe8cfaFWD7p0wVv8fJl3mbzXh33RS+0oW2wvUqiRXiQ69gLWSLpgB5/6sU+r6BlQR0MBILadqTQ==} - deprecated: This package is deprecated. Use the optional chaining (?.) operator instead. + lodash.defaults@4.2.0: + resolution: {integrity: sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==} + + lodash.escape@4.0.1: + resolution: {integrity: sha512-nXEOnb/jK9g0DYMr1/Xvq6l5xMD7GDG55+GSYIYmS0G4tBk/hURD4JR9WCavs04t33WmJx9kCyp9vJ+mr4BOUw==} + + lodash.flattendeep@4.4.0: + resolution: {integrity: sha512-uHaJFihxmJcEX3kT4I23ABqKKalJ/zDrDg0lsFtc1h+3uw49SIJ5beyhx5ExVRti3AvKoOJngIj7xz3oylPdWQ==} lodash.includes@4.3.0: resolution: {integrity: sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==} + lodash.isarguments@3.1.0: + resolution: {integrity: sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==} + lodash.isboolean@3.0.3: resolution: {integrity: sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==} + lodash.isequal@4.5.0: + resolution: {integrity: sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ==} + deprecated: This package is deprecated. Use require('node:util').isDeepStrictEqual instead. + lodash.isinteger@4.0.4: resolution: {integrity: sha512-DBwtEWN2caHQ9/imiNeEA5ys1JoRtRfY3d7V9wkqtbycnAmTvRRmbHKDV4a0EYc678/dia0jrte4tjYwVBaZUA==} @@ -8731,12 +8162,6 @@ packages: resolution: {integrity: sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg==} engines: {node: '>=10'} - lolex@2.7.5: - resolution: {integrity: sha512-l9x0+1offnKKIzYVjyXU2SiwhXDLekRzKyhnbyldPHvC7BvLPVpdNUNR2KeMAiCN2D/kLNttZgQD5WjSxuBx3Q==} - - lolex@5.1.2: - resolution: {integrity: sha512-h4hmjAvHTmd+25JSwrtTIuwbKdwg5NzZVRMLn9saij4SZaepCrTCxPr35H/3bjwfMJtN+t3CX8672UIkglz28A==} - long@5.3.1: resolution: {integrity: sha512-ka87Jz3gcx/I7Hal94xaN2tZEOPoUOEVftkQqZx2EeQRN7LGdfLlI3FvZ+7WDplm+vK2Urx9ULrvSowtdCieng==} @@ -8834,6 +8259,9 @@ packages: md5@2.3.0: resolution: {integrity: sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==} + mdast-util-to-hast@13.2.0: + resolution: {integrity: sha512-QGYKEuUsYT9ykKBCMOEDLsU5JRObWQusAolFMeko/tYPufNkRffBAQjIE+99jbA87xv6FgmjLtwjh9wBWajwAA==} + mdurl@1.0.1: resolution: {integrity: sha512-/sKlQJCBYVY9Ers9hqzKou4H6V5UWc/M59TH2dvkt+84itfnq7uFOMLpOiOS4ujvHP4etln18fmIxA5R5fll0g==} @@ -8880,6 +8308,21 @@ packages: resolution: {integrity: sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==} engines: {node: '>= 0.6'} + micromark-util-character@2.1.1: + resolution: {integrity: sha512-wv8tdUTJ3thSFFFJKtpYKOYiGP2+v96Hvk4Tu8KpCAsTMs6yi+nVmGh1syvSCsaxz45J6Jbw+9DD6g97+NV67Q==} + + micromark-util-encode@2.0.1: + resolution: {integrity: sha512-c3cVx2y4KqUnwopcO9b/SCdo2O67LwJJ/UyqGfbigahfegL9myoEFoDYZgkT7f36T0bLrM9hZTAaAyH+PCAXjw==} + + micromark-util-sanitize-uri@2.0.1: + resolution: {integrity: sha512-9N9IomZ/YuGGZZmQec1MbgxtlgougxTodVwDzzEouPKo3qFWvymFHWcnDi2vzV1ff6kas9ucW+o3yzJK9YB1AQ==} + + micromark-util-symbol@2.0.1: + resolution: {integrity: sha512-vs5t8Apaud9N28kgCrRUdEed4UJ+wWNvicHLPxCa9ENlYuAY31M0ETy5y1vA33YoNPDFTghEbnh6efaE8h4x0Q==} + + micromark-util-types@2.0.2: + resolution: {integrity: sha512-Yw0ECSpJoViF1qTU4DC6NwtC4aWGt1EkzaQB8KPPyCRR8z9TWeV0HbEFGTO+ZY1wB22zmxnJqhPyTpOVCpeHTA==} + micromatch@4.0.8: resolution: {integrity: sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==} engines: {node: '>=8.6'} @@ -8909,10 +8352,6 @@ packages: resolution: {integrity: sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==} engines: {node: '>=6'} - mimic-fn@4.0.0: - resolution: {integrity: sha512-vqiC06CuhBTUdZH+RYl8sFrL096vA45Ok5ISO6sE/Mr1jRbGH4Csnhi8f3wKVl7x8mO4Au7Ir9D3Oyv1VYMFJw==} - engines: {node: '>=12'} - mimic-response@3.1.0: resolution: {integrity: sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==} engines: {node: '>=10'} @@ -8979,14 +8418,12 @@ packages: mlly@1.7.4: resolution: {integrity: sha512-qmdSIPC4bDJXgZTCR7XosJiNKySV7O215tsPtDN9iEO/7q/76b/ijtgRu/+epFXSJhijtTCCGp3DWS549P3xKw==} - mocha@10.8.2: - resolution: {integrity: sha512-VZlYo/WE8t1tstuRmqgeyBgCbJc/lEdopaa+axcKzTBJ+UIdlAB9XnmvTCAH4pwR4ElNInaedhEBmZD8iCSVEg==} - engines: {node: '>= 14.0.0'} - hasBin: true - moment@2.30.1: resolution: {integrity: sha512-uEmtNhbDOrWPFS+hdjFCBfy9f2YoyzRpwcl+DqpC6taX21FzsTLQVbMV/W7PzNSX6x/bhC1zA3c2UQ5NzH6how==} + moo@0.5.2: + resolution: {integrity: sha512-iSAJLHYKnX41mKcJKjqvnAN9sf0LMDTXDEvFv+ffuRR9a1MIuXLjMNL6EsnDHSkKLTWNqQQ5uo61P4EbU4NU+Q==} + mouse-change@1.4.0: resolution: {integrity: sha512-vpN0s+zLL2ykyyUDh+fayu9Xkor5v/zRD9jhSqjRS1cJTGS0+oakVZzNm5n19JvvEj0you+MXlYTpNxUDQUjkQ==} @@ -9016,9 +8453,6 @@ packages: resolution: {integrity: sha512-2eznPJP8z2BFLX50tf0LuODrpINqP1RVIm/CObbTcBRITQgmC/TjcREF1NeTBzIcR5XO/ukWo+YHOjBbFwIupg==} hasBin: true - multimath@2.0.0: - resolution: {integrity: sha512-toRx66cAMJ+Ccz7pMIg38xSIrtnbozk0dchXezwQDMgQmbGpfxjtv68H+L00iFL8hxDaVjrmwAFSb3I6bg8Q2g==} - mumath@3.3.4: resolution: {integrity: sha512-VAFIOG6rsxoc7q/IaY3jdjmrsuX9f15KlRLYTHmixASBZkZEKC1IFqE2BC5CdhXmK6WLM1Re33z//AGmeRI6FA==} deprecated: Redundant dependency in your project. @@ -9037,9 +8471,6 @@ packages: resolution: {integrity: sha512-at/ZndSy3xEGJ8i0ygALh8ru9qy7gWW1cmkaqBN29JmMlIvM//MEO9y1sk/avxuwnPcfhkejkLsuPxH81BrkSg==} engines: {node: '>=0.8.0'} - nan@2.17.0: - resolution: {integrity: sha512-2ZTgtl0nJsO0KQCjEpxcIr5D+Yv90plTitZt9JBfQvVJDS5seMl3FOvsh3+9CoYWXf/1l5OaZzzF6nDm4cagaQ==} - nan@2.20.0: resolution: {integrity: sha512-bk3gXBZDGILuuo/6sKtr0DQmSThYHLtNCdSdXk9YkxD/jK6X2vmCyyXBBxyqZ4XcnzTyYEAThfX3DCEnLf6igw==} @@ -9048,19 +8479,17 @@ packages: engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} hasBin: true + nanoid@3.3.8: + resolution: {integrity: sha512-WNLf5Sd8oZxOm+TzppcYk8gVOgP+l58xNy58D0nbUnOxOWRWvlcCV4kUF7ltmI6PsrLl/BgKEyS4mqsGChFN0w==} + engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} + hasBin: true + napi-build-utils@2.0.0: resolution: {integrity: sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA==} native-promise-only@0.8.1: resolution: {integrity: sha512-zkVhZUA3y8mbz652WrL5x0fB0ehrBkulWT3TomAQ9iDtyXZvzKeEA6GPxAItBYeNYl5yngKRX612qHOhvMkDeg==} - nats.ws@1.30.3: - resolution: {integrity: sha512-aM77V2SEc+B6lbxCMZK3qfRy4jg8pmHj+wZzQKDiDIQYhLPj6U2NSHHBex0syj72Ayzl4uR5Lp3aKXTaVLbRpw==} - - nats@2.29.3: - resolution: {integrity: sha512-tOQCRCwC74DgBTk4pWZ9V45sk4d7peoE2njVprMRCBXrhJ5q5cYM7i6W+Uvw2qUrcfOSnuisrX7bEx3b3Wx4QA==} - engines: {node: '>= 14.0.0'} - natural-compare@1.4.0: resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==} @@ -9068,6 +8497,10 @@ packages: resolution: {integrity: sha512-zIdGUrPRFTUELUvr3Gmc7KZ2Sw/h1PiVM0Af/oHB6zgnV1ikqSfRk+TOufi79aHYCW3NiOXmr1BP5nWbzojLaA==} hasBin: true + nearley@2.20.1: + resolution: {integrity: sha512-+Mc8UaAebFzgV+KpI5n7DasuuQCHA89dmwm7JXw3TV43ukfNQ9DnBH3Mdb2g/I4Fdxc26pwimBWvjIw0UAILSQ==} + hasBin: true + needle@2.9.1: resolution: {integrity: sha512-6R9fqJ5Zcmf+uYaFgdIHmLwNldn5HbK8L5ybn7Uz+ylX/rnOsSp1AHcvQSrCaFN+qNM1wpymHqD7mVasEOlHGQ==} engines: {node: '>= 4.4.x'} @@ -9128,13 +8561,6 @@ packages: sass: optional: true - nise@1.5.3: - resolution: {integrity: sha512-Ymbac/94xeIrMf59REBPOv0thr+CJVFMhrlAkW/gjCIE58BGQdCj0x7KRCb3yz+Ga2Rz3E9XXSvUyyxqqhjQAQ==} - - nkeys.js@1.1.0: - resolution: {integrity: sha512-tB/a0shZL5UZWSwsoeyqfTszONTt4k2YS0tuQioMOD180+MbombYVgzDUYHlx+gejYK6rgf08n/2Df99WY0Sxg==} - engines: {node: '>=10.0.0'} - no-case@3.0.4: resolution: {integrity: sha512-fgAN3jGAh+RoxUGZHTSOLJIqUc2wmoBwGR4tbpNAKmmovFoWq0OdRkb0VkldReO2a2iBT/OEulG9XSUc10r3zg==} @@ -9142,14 +8568,12 @@ packages: resolution: {integrity: sha512-c5XK0MjkGBrQPGYG24GBADZud0NCbznxNx0ZkS+ebUTrmV1qTDxPxSL8zEAPURXSbLRWVexxmP4986BziahL5w==} engines: {node: '>=10'} - node-addon-api@6.1.0: - resolution: {integrity: sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA==} - node-addon-api@7.1.1: resolution: {integrity: sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ==} - node-cjsx@2.0.0: - resolution: {integrity: sha512-WstoLFETzse3jiXc7tcL9BI81k7L9Rgi4SjAWe691TZiooZRUpuCmIcZYcYzkDFtmg8FJkuK9WDFsz6llmBHcg==} + node-addon-api@8.3.1: + resolution: {integrity: sha512-lytcDEdxKjGJPTLEfW4mYMigRezMlyJY8W4wxJK8zE533Jlb8L8dRuObJFWg2P+AuOIxoCgKF+2Oq4d4Zd0OUA==} + engines: {node: ^18 || ^20 || >= 21} node-cleanup@2.1.2: resolution: {integrity: sha512-qN8v/s2PAJwGUtr1/hYTpNKlD6Y9rc4p8KSmJXyGdYGZsDGKXrGThikLFP9OCHFeLeEpQzPwiAtdIvBLqm//Hw==} @@ -9177,18 +8601,10 @@ packages: encoding: optional: true - node-fetch@3.3.2: - resolution: {integrity: sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==} - engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} - node-forge@1.3.1: resolution: {integrity: sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA==} engines: {node: '>= 6.13.0'} - node-gyp-build@4.5.0: - resolution: {integrity: sha512-2iGbaQBV+ITgCz76ZEjmhUKAKVf7xfY1sRl4UiKQspfZMH2h06SyhNsnSVy50cwkFQDGLyif6m/6uFXHkOZ6rg==} - hasBin: true - node-int64@0.4.0: resolution: {integrity: sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==} @@ -9243,6 +8659,9 @@ packages: normalize-wheel@1.0.1: resolution: {integrity: sha512-1OnlAPZ3zgrk8B91HyRj+eVv+kS5u+Z0SCsak6Xil/kmgEia50ga7zfkumayonZrImffAxPU/5WcyGhzetHNPA==} + notepack.io@3.0.1: + resolution: {integrity: sha512-TKC/8zH5pXIAMVQio2TvVDTtPRX+DJPHDqjRbxogtFiByHyzKmy96RA0JtCQJ+WouyyL4A10xomQzgbUT+1jCg==} + nouislider@15.4.0: resolution: {integrity: sha512-AV7UMhGhZ4Mj6ToMT812Ib8OJ4tAXR2/Um7C4l4ZvvsqujF0WpQTpqqHJ+9xt4174R7ueQOUrBR4yakJpAIPCA==} @@ -9250,10 +8669,6 @@ packages: resolution: {integrity: sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==} engines: {node: '>=8'} - npm-run-path@5.3.0: - resolution: {integrity: sha512-ppwTtiJZq0O/ai0z7yfudtBpWIoxM8yE6nHi1X47eFR2EWORqfbu6CnPlNsjeN683eT0qG6H/Pyf9fCcvjnnnQ==} - engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} - nth-check@2.1.1: resolution: {integrity: sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==} @@ -9275,10 +8690,6 @@ packages: resolution: {integrity: sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw==} engines: {node: '>= 6'} - object-inspect@1.13.2: - resolution: {integrity: sha512-IRZSRuzJiynemAXPYtPe5BoI/RESNYR7TYm50MC5Mqbd3Jmw5y790sErYw3V6SryFJD64b74qQQs9wn5Bg/k3g==} - engines: {node: '>= 0.4'} - object-inspect@1.13.4: resolution: {integrity: sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==} engines: {node: '>= 0.4'} @@ -9310,9 +8721,6 @@ packages: obuf@1.1.2: resolution: {integrity: sha512-PX1wu0AmAdPqOL1mWhqmlOd8kOIZQwGZw6rh7uby9fTc5lhaOWFLX3I6R1hrF9k3zUY40e6igsLGkDXK92LJNg==} - octicons@3.5.0: - resolution: {integrity: sha512-jIjd+/oT46YgOK2SZbicD8vIGkinUwpx7HRm6okT3dU5fZQO/sEbMOKSfLxLhJIuI2KRlylN9nYfKiuM4uf+gA==} - ollama@0.5.15: resolution: {integrity: sha512-TSaZSJyP7MQJFjSmmNsoJiriwa3U+/UJRw6+M8aucs5dTsaWNZsBIGpDb5rXnW6nXxJBB/z79gZY8IaiIQgelQ==} @@ -9338,10 +8746,6 @@ packages: resolution: {integrity: sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==} engines: {node: '>=6'} - onetime@6.0.0: - resolution: {integrity: sha512-1FlR+gjXK7X+AsAHso35MnyN5KqGwJRi/31ft6x0M194ht7S+rWAvd7PHss9xSKMzE0asv1pyIHaJYq+BbacAQ==} - engines: {node: '>=12'} - open@10.1.2: resolution: {integrity: sha512-cxN6aIDPz6rm8hbebcP7vrQNhvRcveZoJU72Y7vskh4oIm+BZwBECnx5nTmrlres1Qapvx27Qo1Auukpf8PKXw==} engines: {node: '>=18'} @@ -9500,6 +8904,9 @@ packages: parse5@6.0.1: resolution: {integrity: sha512-Ofn/CTFzRGTTxwpNEs9PP93gXShHcTq255nzRYSKe8AkVpZY7e1fpmTfOyoIvjP5HG7Z2ZM7VS9PPhQGW2pOpw==} + parse5@7.2.1: + resolution: {integrity: sha512-BuBYQYlv1ckiPdQi/ohiivi9Sagc9JG+Ozs0r7b/0iK3sKmrb0b9FdWdBbOdx6hBCM/F9Ir82ofnBhtZOjCRPQ==} + parse5@7.3.0: resolution: {integrity: sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==} @@ -9537,10 +8944,6 @@ packages: resolution: {integrity: sha512-KSk6IJ15RoxuGq7D1UKK/8qKhNfzbLeLrG3gkLZ7p4A6DBCcv7xpyQwuXtWdpyR0+E0mwkpjY1VfPOhxQrKzdQ==} engines: {node: '>= 0.4.0'} - passport-ldapauth@3.0.1: - resolution: {integrity: sha512-TRRx3BHi8GC8MfCT9wmghjde/EGeKjll7zqHRRfGRxXbLcaDce2OftbQrFG7/AWaeFhR6zpZHtBQ/IkINdLVjQ==} - engines: {node: '>=0.8.0'} - passport-oauth1@1.3.0: resolution: {integrity: sha512-8T/nX4gwKTw0PjxP1xfD0QhrydQNakzeOpZ6M5Uqdgz9/a/Ag62RmJxnZQ4LkbdXGrRehQHIAHNAu11rCP46Sw==} engines: {node: '>= 0.4.0'} @@ -9602,10 +9005,6 @@ packages: resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==} engines: {node: '>=8'} - path-key@4.0.0: - resolution: {integrity: sha512-haREypq7xkM7ErfgIyA0z+Bj4AGKlMSdlQE2jvJo6huWD1EdkKYV+G/T4nq0YEF2vgTT8kqMFKo1uHn950r4SQ==} - engines: {node: '>=12'} - path-parse@1.0.7: resolution: {integrity: sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==} @@ -9616,9 +9015,6 @@ packages: path-to-regexp@0.1.12: resolution: {integrity: sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==} - path-to-regexp@1.9.0: - resolution: {integrity: sha512-xIp7/apCFJuUHdDLWe8O1HIkb0kQrOMb/0u6FXQjemHn/ii5LrIzU6bdECnsiTF/GjZkMEKg1xdiZwNqDYlZ6g==} - path-to-regexp@8.2.0: resolution: {integrity: sha512-TdrF7fW9Rphjq4RjrW0Kp2AW0Ahwu9sRGTkS6bvDi0SCwZlEZYmcfDbEsTz8RVk0EHIS/Vd1bv3JhG+1xZuAyQ==} engines: {node: '>=16'} @@ -9645,10 +9041,9 @@ packages: resolution: {integrity: sha512-/Y3fcFrXEAsMjJXeL9J8+ZG9U01LbuWaYypvDW2ycW1jL269L3js3DVBjDJ0Up9Np1uqDXsDrRihHANhZOlwdQ==} engines: {node: '>=20'} - pegjs@0.10.0: - resolution: {integrity: sha512-qI5+oFNEGi3L5HAxDwN2LA4Gg7irF70Zs25edhjld9QemOgp0CbvMtbFcMvFtEo1OityPrcCzkQFB8JP/hxgow==} - engines: {node: '>=0.10'} - hasBin: true + peek-readable@7.0.0: + resolution: {integrity: sha512-nri2TO5JE3/mRryik9LlHFT53cgHfRK0Lt0BAZQXku/AW3E6XLt2GaY8siWi7dvW/m1z0ecn+J+bpDa9ZN3IsQ==} + engines: {node: '>=18'} performance-now@2.1.0: resolution: {integrity: sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow==} @@ -9690,9 +9085,6 @@ packages: pgpass@1.0.5: resolution: {integrity: sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==} - pica@7.1.1: - resolution: {integrity: sha512-WY73tMvNzXWEld2LicT9Y260L43isrZ85tPuqRyvtkljSDLmnNFQmZICt4xUJMVulmcc6L9O7jbBrtx3DOz/YQ==} - pick-by-alias@1.2.0: resolution: {integrity: sha512-ESj2+eBxhGrcA1azgHs7lARG5+5iLakc/6nlfbpjcLl00HuuUOIuORhYXN4D1HfvMSKuVtFQjAlnwi1JHEeDIw==} @@ -9710,10 +9102,6 @@ packages: resolution: {integrity: sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==} engines: {node: '>=12'} - pidusage@1.2.0: - resolution: {integrity: sha512-OGo+iSOk44HRJ8q15AyG570UYxcm5u+R99DI8Khu8P3tKGkVu5EZX4ywHglWSTMNNXQ274oeGpYrvFEhDIFGPg==} - engines: {node: '>=0.12'} - pify@2.3.0: resolution: {integrity: sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog==} engines: {node: '>=0.10.0'} @@ -9748,10 +9136,6 @@ packages: pkg-types@2.1.0: resolution: {integrity: sha512-wmJwA+8ihJixSoHKxZJRBQG1oY8Yr9pGLzRmSsNms0iNWyHHAlZCa7mmKiFR10YPZuz/2k169JiS/inOjBCZ2A==} - pkginfo@0.4.1: - resolution: {integrity: sha512-8xCNE/aT/EXKenuMDZ+xTVwkT8gsoHN2z/Q29l80u0ppGEXVvsKRzNMbtKhg8LS8k1tJLAHHylf6p4VFmP6XUQ==} - engines: {node: '>= 0.4.0'} - playwright-core@1.51.1: resolution: {integrity: sha512-/crRMj8+j/Nq5s8QcvegseuyeZPxpQCZb6HNk3Sos3BlZyAknRjoyJPFWkpNn8v0+P3WiwqFF8P+zQo4eqiNuw==} engines: {node: '>=18'} @@ -9784,10 +9168,6 @@ packages: port-get@1.0.4: resolution: {integrity: sha512-B8RcNfc8Ld+7C31DPaKIQz2aO9dqIs+4sUjhxJ2TSjEaidwyxu05WBbm08FJe+qkVvLiQqPbEAfNw1rB7JbjtA==} - portfinder@1.0.37: - resolution: {integrity: sha512-yuGIEjDAYnnOex9ddMnKZEMFE0CcGo6zbfzDklkmT1m5z734ss6JMzN9rNB3+RR7iS+F10D4/BVIaXOyh8PQKw==} - engines: {node: '>= 10.12'} - possible-typed-array-names@1.1.0: resolution: {integrity: sha512-/+5VFTchJDoVj3bhoqi6UeymcD00DAwb1nJwamzPvHEszJ4FpF6SNNbUbOS8yI56qHzdV8eK0qEfOSiodkTdxg==} engines: {node: '>= 0.4'} @@ -9901,6 +9281,9 @@ packages: primus@8.0.9: resolution: {integrity: sha512-gWsd6pWHAHGfyArl6DQU9iCAp4bAgFrintDpFbyA2r0wdzJ2n9SsffSaFqOKYZeE9wqKcBepnwBGoFKzNybqMA==} + printable-characters@1.0.42: + resolution: {integrity: sha512-dKp+C4iXWK4vVYZmYSd0KBH5F/h1HoZRsbJ82AVKRO3PEo8L4lBS/vLwhVtpwwuYcoIsVY+1JYKR268yn480uQ==} + probe-image-size@7.2.3: resolution: {integrity: sha512-HubhG4Rb2UH8YtV4ba0Vp5bQ7L78RTONYu/ujmCu5nBI8wGv24s4E9xSKBi0N1MowRpxk76pFCpJtW0KPzOK0w==} @@ -9930,14 +9313,13 @@ packages: property-information@6.5.0: resolution: {integrity: sha512-PgTgs/BlvHxOu8QuEN7wi5A0OmXaBcHpmCSTehcs6Uuu9IkDIEo13Hy7n898RHfrQ49vKCoGeWZSaAK01nwVig==} + property-information@7.1.0: + resolution: {integrity: sha512-TwEZ+X+yCJmYfL7TPUOcvBZ4QfoT5YenQiJuX//0th53DE6w0xxLEtfK3iyryQFddXuvkIk51EEgrJQ0WJkOmQ==} + proto3-json-serializer@2.0.2: resolution: {integrity: sha512-SAzp/O4Yh02jGdRc+uIrGoe87dkN/XtwxfZ4ZyafJHymd79ozp5VG5nyZ7ygqPM5+cpLDjjGnYFUkngonyDPOQ==} engines: {node: '>=14.0.0'} - proto3-json-serializer@3.0.0: - resolution: {integrity: sha512-mHPIc7zaJc26HMpgX5J7vXjliYv4Rnn5ICUyINudz76iY4zFMQHTaQXrTFn0EoHnRsLD6BE+OuHhQHFUU93I9A==} - engines: {node: '>=18'} - protobufjs@7.4.0: resolution: {integrity: sha512-mRUWCc3KUU4w1jU8sGxICXH/gNS94DvI1gxqDvBzhj1JpcsimQkYiOJfwsPUykUI5ZaspFbSgmBLER8IrQ3tqw==} engines: {node: '>=12.0.0'} @@ -9980,10 +9362,6 @@ packages: quansync@0.2.10: resolution: {integrity: sha512-t41VRkMYbkHyCYmOvx/6URnN80H7k4X0lLdBMGsz+maAwrJQYB1djpV6vHrQIBE0WBSGqhtEHrK9U3DWWH8v7A==} - query-string@3.0.3: - resolution: {integrity: sha512-51caZjRlfBSfcCvFT5OKJqY7az8z05qAHx1nHydQyEYIxOThv1BLTYt+T+usyJpPCsoGQDQxCdDzZ7BbIZtitw==} - engines: {node: '>=0.10.0'} - querystringify@2.2.0: resolution: {integrity: sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==} @@ -10006,6 +9384,13 @@ packages: raf@3.4.1: resolution: {integrity: sha512-Sq4CW4QhwOHE8ucn6J34MqtZCeWFP2aQSmrlroYgqAV1PjStIhJXxYuTgUIfkEk7zTLjmIjLmU5q+fbD1NnOJA==} + railroad-diagrams@1.0.0: + resolution: {integrity: sha512-cz93DjNeLY0idrCNOH6PviZGRN9GJhsdm9hpn1YCS879fj4W+x5IFJhhkRZcwVgMmFF7R82UA/7Oh+R8lLZg6A==} + + randexp@0.4.6: + resolution: {integrity: sha512-80WNmd9DA0tmZrw9qQa62GPPWfuXJknrmVmLcxvq4uZBdYqb1wYoKTmnlGUchvVWe0XiLupYkBoXVOxz3C8DYQ==} + engines: {node: '>=0.12'} + random-bytes@1.0.0: resolution: {integrity: sha512-iv7LhNVO047HzYR3InF6pUcUsPQiHTM1Qal51DcGSuZFBil1aBBWG5eHPNek7bvILMaYJ/8RU1e8w1AMdHmLQQ==} engines: {node: '>= 0.8'} @@ -10292,11 +9677,6 @@ packages: react: ^16.13.1 || ^17.0.0 || ^18.0.0 || ^19.0.0 react-dom: ^16.13.1 || ^17.0.0 || ^18.0.0 || ^19.0.0 - react-async-script@1.2.0: - resolution: {integrity: sha512-bCpkbm9JiAuMGhkqoAiC0lLkb40DJ0HOEJIku+9JDjxX3Rcs+ztEOG13wbrOskt3n2DTrjshhaQ/iay+SnGg5Q==} - peerDependencies: - react: '>=16.4.1' - react-base16-styling@0.9.1: resolution: {integrity: sha512-1s0CY1zRBOQ5M3T61wetEpvQmsYSNtWEcdYzyZNxKa8t7oDvaOn9d21xrGezGAHFWLM7SHcktPuPTrvoqxSfKw==} @@ -10345,12 +9725,7 @@ packages: resolution: {integrity: sha512-kLQqpz/77m8+trpBwzqcxNtvWZYoZ/YO6Vm2cVTHW8hs80BWUfDpC7RDwuAvpswwtSYApWfaSpIDFWAIBNIYxQ==} peerDependencies: react: ^16.3 || ^17.0 || ^18.0 || ^19.0 - react-dom: ^17.0 || ^18.0 || ^19.0 - - react-google-recaptcha@2.1.0: - resolution: {integrity: sha512-K9jr7e0CWFigi8KxC3WPvNqZZ47df2RrMAta6KmRoE4RUi7Ys6NmNjytpXpg4HI/svmQJLKR+PncEPaNJ98DqQ==} - peerDependencies: - react: '>=16.4.1' + react-dom: ^17.0 || ^18.0 || ^19.0 react-helmet@6.1.0: resolution: {integrity: sha512-4uMzEY9nlDlgxr61NL3XbKRy1hEkXmKNXhjbAIOVw5vcFrsdYbH2FEwcNyWvWinl103nXgzYNlns9ca+8kFiWw==} @@ -10501,6 +9876,14 @@ packages: resolution: {integrity: sha512-6tDA8g98We0zd0GvVeMT9arEOnTw9qM03L9cJXaCjrip1OO764RDBLBfrB4cwzNGDj5OA5ioymC9GkizgWJDUg==} engines: {node: '>=8'} + redis-errors@1.2.0: + resolution: {integrity: sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==} + engines: {node: '>=4'} + + redis-parser@3.0.0: + resolution: {integrity: sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==} + engines: {node: '>=4'} + redux@4.2.1: resolution: {integrity: sha512-LAUYz4lc+Do8/g7aeRa8JkyDErK6ekstQaqWQrNRW//MY1TvCEpMtpTWvlQ+FPbWCx+Xixu/6SHt5N0HR+SB4w==} @@ -10539,17 +9922,17 @@ packages: regl@2.1.1: resolution: {integrity: sha512-+IOGrxl3FZ8ZM9ixCWQZzFRiRn7Rzn9bu3iFHwg/yz4tlOUQgbO4PHLgG+1ZT60zcIV8tief6Qrmyl8qcoJP0g==} - rehype-parse@8.0.5: - resolution: {integrity: sha512-Ds3RglaY/+clEX2U2mHflt7NlMA72KspZ0JLUJgBBLpRddBcEw3H8uYZQliQriku22NZpYMfjDdSgHcjxue24A==} + rehype-parse@9.0.1: + resolution: {integrity: sha512-ksCzCD0Fgfh7trPDxr2rSylbwq9iYDkSn8TCDmEJ49ljEUBxDVCzCHv7QNzZOfODanX4+bWQ4WZqLCRWYLfhag==} - rehype-prism-plus@1.6.3: - resolution: {integrity: sha512-F6tn376zimnvy+xW0bSnryul+rvVL7NhDIkavc9kAuzDx5zIZW04A6jdXPkcFBhojcqZB8b6pHt6CLqiUx+Tbw==} + rehype-prism-plus@2.0.0: + resolution: {integrity: sha512-FeM/9V2N7EvDZVdR2dqhAzlw5YI49m9Tgn7ZrYJeYHIahM6gcXpH0K1y2gNnKanZCydOMluJvX2cB9z3lhY8XQ==} - rehype-stringify@9.0.4: - resolution: {integrity: sha512-Uk5xu1YKdqobe5XpSskwPvo1XeHUUucWEQSl8hTrXt5selvca1e8K1EZ37E6YoZ4BT8BCqCdVfQW7OfHfthtVQ==} + rehype-stringify@10.0.1: + resolution: {integrity: sha512-k9ecfXHmIPuFVI61B9DeLPN0qFHfawM6RsuX48hoqlaKSF61RskNjSm1lI8PhBEM0MRdLxVVm4WmTqJQccH9mA==} - rehype@12.0.1: - resolution: {integrity: sha512-ey6kAqwLM3X6QnMDILJthGvG1m1ULROS9NT4uG9IDCuv08SFyLlreSuvOa//DgEvbXx62DS6elGVqusWhRUbgw==} + rehype@13.0.2: + resolution: {integrity: sha512-j31mdaRFrwFRUIlxGeuPXXKWQxet52RBQRvCmzl5eCefn/KGbomK5GMHNMsOJf55fgo3qw5tST5neDuarDYR2A==} relateurl@0.2.7: resolution: {integrity: sha512-G08Dxvm4iDN3MLM0EsP62EDV9IuhXPR6blNz6Utcp7zyV3tr4HVNINt6MpaRWbxoOHT3Q7YN2P+jaHX8vUbgog==} @@ -10570,10 +9953,6 @@ packages: resolution: {integrity: sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==} engines: {node: '>=0.10.0'} - require-reload@0.2.2: - resolution: {integrity: sha512-ElZsgUSIyQKuS8Db4t/w30ut7TXWPzEhvBEVWzMHMTHeokHABEiF+oABX/rSp9nEhm+loT/ziZC+/7PQn7+7eA==} - engines: {node: '>=0.6.21'} - requires-port@1.0.0: resolution: {integrity: sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==} @@ -10605,6 +9984,11 @@ packages: resolve@0.6.3: resolution: {integrity: sha512-UHBY3viPlJKf85YijDUcikKX6tmF4SokIDp518ZDVT92JNDcG5uKIthaT/owt3Sar0lwtOafsQuwrg22/v2Dwg==} + resolve@1.22.10: + resolution: {integrity: sha512-NPRy+/ncIMeDlTAsuqwKIiferiawhefFJtkNSW0qZJEqMEb+qBt/77B/jGeeek+F0uOeN05CDa6HXbbIgtVX4w==} + engines: {node: '>= 0.4'} + hasBin: true + resolve@1.22.8: resolution: {integrity: sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==} hasBin: true @@ -10617,14 +10001,14 @@ packages: resolution: {integrity: sha512-l+sSefzHpj5qimhFSE5a8nufZYAM3sBSVMAPtYkmC+4EH2anSGaEMXSD0izRQbu9nfyQ9y5JrVmp7E8oZrUjvA==} engines: {node: '>=8'} + ret@0.1.15: + resolution: {integrity: sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg==} + engines: {node: '>=0.12'} + retry-request@7.0.2: resolution: {integrity: sha512-dUOvLMJ0/JJYEn8NrpOaGNE7X3vpI5XlZS/u0ANjqtcZVKnIxP7IgCFwrKTxENw29emmwug53awKtaMm4i9g5w==} engines: {node: '>=14'} - retry-request@8.0.0: - resolution: {integrity: sha512-dJkZNmyV9C8WKUmbdj1xcvVlXBSvsUQCkg89TCK8rD72RdSn9A2jlXlS2VuYSTHoPJjJEfUHhjNYrlvuksF9cg==} - engines: {node: '>=18'} - retry@0.13.1: resolution: {integrity: sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==} engines: {node: '>= 4'} @@ -10661,6 +10045,9 @@ packages: roughjs@4.6.6: resolution: {integrity: sha512-ZUz/69+SYpFN/g/lUlo2FXcIjRkSu3nDarreVdGGndHEBJ6cXPdKguS8JGxwj5HA5xIbVKSmLgr5b3AWxtRfvQ==} + rst-selector-parser@2.2.3: + resolution: {integrity: sha512-nDG1rZeP6oFTLN6yNDV/uiAvs1+FS/KlrEwh7+y7dpuApDBy6bI2HTBcc0/V8lv9OTqfyD34eF7au2pm8aBbhA==} + run-applescript@7.0.0: resolution: {integrity: sha512-9by4Ij99JUr/MCFBUkDKLWK3G9HVXmabKz9U5MlIAIuvuzkiOicRYs8XJLxX+xahD+mLiiCYDqF9dKAgtzKP1A==} engines: {node: '>=18'} @@ -10706,10 +10093,6 @@ packages: safer-buffer@2.1.2: resolution: {integrity: sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==} - samsam@1.3.0: - resolution: {integrity: sha512-1HwIYD/8UlOtFS3QO3w7ey+SdSDFE4HRNLZoZRYVQefrOY3l17epswImeB1ijgJFQJodIaHcwkp3r/myBjFVbg==} - deprecated: This package has been deprecated in favour of @sinonjs/samsam - sanitize-html@2.17.0: resolution: {integrity: sha512-dLAADUSS8rBwhaevT12yCezvioCA+bmUTPH/u57xKPT8d++voeYE6HeluA/bPbQ15TwDBG2ii+QZIEmYx8VdxA==} @@ -10845,10 +10228,6 @@ packages: shallowequal@1.1.0: resolution: {integrity: sha512-y0m1JoUZSlPAjXVtPPW70aZWfIL/dSP7AFkRnniLCrK/8MDKog3TySTBmckD+RObVxH0v4Tox67+F14PdED2oQ==} - sharp@0.32.6: - resolution: {integrity: sha512-KyLTWwgcR9Oe4d9HwCwNM2l7+J0dUQwn/yf7S0EnTtb0eVS4RxO0eUSvxPtzT4F3SY+C4K6fqdv/DO27sJ/v/w==} - engines: {node: '>=14.15.0'} - shebang-command@2.0.0: resolution: {integrity: sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==} engines: {node: '>=8'} @@ -10864,26 +10243,9 @@ packages: resolution: {integrity: sha512-ObmnIF4hXNg1BqhnHmgbDETF8dLPCggZWBjkQfhZpbszZnYur5DUljTcCHii5LC3J5E0yeO/1LIMyH+UvHQgyw==} engines: {node: '>= 0.4'} - should-equal@0.5.0: - resolution: {integrity: sha512-y+N0u99fC0EjLbWovcpwSziVnzhEEmJo7//4DKM5Ioc0FfqDKnDNNnDXjWCU2G9SoGLnJnfdq0NiOoZFoDBEqA==} - - should-format@0.3.1: - resolution: {integrity: sha512-CBN7ZbI2QpfNEmex46Y45YanmvAgBWNhUsKxFAsvaahNIcFbezERyQJwwlI+xFyHXjDHQHn9/Fbs81bonjqmIg==} - should-proxy@1.0.4: resolution: {integrity: sha512-RPQhIndEIVUCjkfkQ6rs6sOR6pkxJWCNdxtfG5pP0RVgUYbK5911kLTF0TNcCC0G3YCGd492rMollFT2aTd9iQ==} - should-sinon@0.0.3: - resolution: {integrity: sha512-jav6kEQ4BcbXGgLl+NuHmGkO23W+XTV5ovBftVSPSuiMAjQisU5o1NCU71HafZwGF5+HQiFxiEcQlpspnonVwQ==} - peerDependencies: - should: '>= 4.x' - - should-type@0.2.0: - resolution: {integrity: sha512-ixbc1p6gw4W29fp4MifFynWVQvuqfuZjib+y1tWezbjinoXu0eab/rXxLDP6drfZXlz6lZBwuzHJrs/BjLCLuQ==} - - should@7.1.1: - resolution: {integrity: sha512-llSOcffBvYZGDZk6xie5jazh1RonS/MuknPrZOOu/uqcUFBGHFhWo+2fseOekmylqyZu9BiQYkAp3zeQFiQFmA==} - side-channel-list@1.0.0: resolution: {integrity: sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA==} engines: {node: '>= 0.4'} @@ -10926,10 +10288,6 @@ packages: simple-wcswidth@1.0.1: resolution: {integrity: sha512-xMO/8eNREtaROt7tJvWJqHBDTMFN4eiQ5I4JRMuilwfnFcV5W9u7RUkueNkdw0jPqGMX36iCywelS5yilTuOxg==} - sinon@4.5.0: - resolution: {integrity: sha512-trdx+mB0VBBgoYucy6a9L7/jfQOmvGeaKZT4OOJ+lPAtI8623xyGr8wLiE4eojzBS8G9yXbhx42GHUOVLr4X2w==} - deprecated: 16.1.1 - sirv@2.0.4: resolution: {integrity: sha512-94Bdh3cC2PKrbgSOUqTiGPWVZeSiXfKOVZNJniWoqrWrRkB1CJzBU3NEbiTsPcYy1lDsANA/THzS+9WBiy5nfQ==} engines: {node: '>= 10'} @@ -10956,6 +10314,21 @@ packages: resolution: {integrity: sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==} engines: {node: '>= 6.0.0', npm: '>= 3.0.0'} + socket.io-adapter@2.5.5: + resolution: {integrity: sha512-eLDQas5dzPgOWCk9GuuJC2lBqItuhKI4uxGgo9aIV7MYbk2h9Q6uULEh8WBzThoI7l+qU9Ast9fVUmkqPP9wYg==} + + socket.io-client@4.8.1: + resolution: {integrity: sha512-hJVXfu3E28NmzGk8o1sHhN3om52tRvwYeidbj7xKy2eIIse5IoKX3USlS6Tqt3BHAtflLIkCQBkzVrEEfWUyYQ==} + engines: {node: '>=10.0.0'} + + socket.io-parser@4.2.4: + resolution: {integrity: sha512-/GbIKmo8ioc+NIWIhwdecY0ge+qVBSMdgxGygevmdHj24bsfgtCmcUUcQ5ZzcylGFHsN3k4HB4Cgkl96KVnuew==} + engines: {node: '>=10.0.0'} + + socket.io@4.8.1: + resolution: {integrity: sha512-oZ7iUCxph8WYRHHcjBEc9unw3adt5CmSNlppj/5Q4k2RIrhl8Z5yY2Xr4j9zj0+wzVZ0bxmYoGSzKJnRl6A4yg==} + engines: {node: '>=10.2.0'} + sockjs@0.3.24: resolution: {integrity: sha512-GJgLTZ7vYb/JtPSSZ10hsOYIvEYsjbNU+zPdIHcUaWVNUEPivzxku31865sSSud0Da0W4lEeOPlmw93zLQchuQ==} @@ -11036,6 +10409,9 @@ packages: stackframe@1.3.4: resolution: {integrity: sha512-oeVtt7eWQS+Na6F//S4kJ2K2VbRlS9D43mAlMyVpVWovy9o+jfgH8O9agzANzaiLjclA0oYzUXEM4PurhSUChw==} + standard-as-callback@2.1.0: + resolution: {integrity: sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==} + static-eval@2.1.1: resolution: {integrity: sha512-MgWpQ/ZjGieSVB3eOJVs4OA2LT/q1vx98KPCTTQPzq/aLr0YUXTsgryTXr4SLfR0ZfUUCiedM9n/ABeDIyy4mA==} @@ -11066,13 +10442,6 @@ packages: resolution: {integrity: sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==} engines: {node: '>=10.0.0'} - streamx@2.22.0: - resolution: {integrity: sha512-sLh1evHOzBy/iWRiR6d1zRcLao4gGZr3C1kzNz4fopCOKJb6xD9ub8Mpi9Mr1R6id5o43S+d93fI48UC5uM9aw==} - - strict-uri-encode@1.1.0: - resolution: {integrity: sha512-R3f198pcvnB+5IpnBlRkphuE9n46WyVl8I39W/ZUTZLz4nqSP/oLYUrcnJrw462Ds8he4YKMov2efsTIw1BDGQ==} - engines: {node: '>=0.10.0'} - string-convert@0.2.1: resolution: {integrity: sha512-u/1tdPl4yQnPBjnVrmdLo9gtuLvELKsAoRapekWggdiQNvvvum+jYF329d84NAa660KQw7pB2n36KrIKVoXa3A==} @@ -11142,10 +10511,6 @@ packages: resolution: {integrity: sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==} engines: {node: '>=6'} - strip-final-newline@3.0.0: - resolution: {integrity: sha512-dOESqjYr96iWYylGObzd39EuNTa5VJxyvVAEm5Jnh7KGo75V43Hk1odPQkNDyXNmUR6k+gEiDVXnjB8HJ3crXw==} - engines: {node: '>=12'} - strip-indent@3.0.0: resolution: {integrity: sha512-laJTa3Jb+VQpaC6DseHhF7dXVqHTfJPCRDaEbid/drOhgitgYku/letMUqOXFoWV0zIIUbjpdH2t+tYj4bQMRQ==} engines: {node: '>=8'} @@ -11168,8 +10533,8 @@ packages: strongly-connected-components@1.0.1: resolution: {integrity: sha512-i0TFx4wPcO0FwX+4RkLJi1MxmcTv90jNZgxMu9XRnMXMeFUY1VJlIoXpZunPUvUUqbCT1pg5PEkFqqpcaElNaA==} - strtok3@10.3.1: - resolution: {integrity: sha512-3JWEZM6mfix/GCJBBUrkA8p2Id2pBkyTkVCJKto55w080QBKZ+8R171fGrbiSp+yMO/u6F8/yUh7K4V9K+YCnw==} + strtok3@10.2.2: + resolution: {integrity: sha512-Xt18+h4s7Z8xyZ0tmBoRmzxcop97R4BAh+dXouUDCYn+Em+1P3qpkUfI5ueWLT8ynC5hZ+q4iPEmGG1urvQGBg==} engines: {node: '>=18'} stubs@3.0.0: @@ -11234,10 +10599,6 @@ packages: resolution: {integrity: sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==} engines: {node: '>=10'} - supports-color@9.4.0: - resolution: {integrity: sha512-VL+lNrEoIXww1coLPOmiEmK/0sGigko5COxI09KzHc2VJXJsQ37UaQ+8quuxjDeA7+KnLGTWRyOXSLLR2Wb4jw==} - engines: {node: '>=12'} - supports-hyperlinks@2.2.0: resolution: {integrity: sha512-6sXEzV5+I5j8Bmq9/vUphGRM/RJNT9SCURJLjwfOg51heRtguGWDzcaBlgAzKhQa0EVNpPEKzQuBwZ8S8WaCeQ==} engines: {node: '>=8'} @@ -11263,10 +10624,6 @@ packages: resolution: {integrity: sha512-9kY+CygyYM6j02t5YFHbNz2FN5QmYGv9zAjVp4lCDjlCw7amdckXlEt/bjMhUIfj4ThGRE4gCUH5+yGnNuPo5A==} engines: {node: '>=10.0.0'} - tapable@2.2.1: - resolution: {integrity: sha512-GNzQvQTOIP6RyTfE2Qxb8ZVlNmw0n88vp1szwWRimP02mnTsx3Wtn5qRdqY9w2XduFNUgvOwhNnQsjwCp+kqaQ==} - engines: {node: '>=6'} - tapable@2.2.2: resolution: {integrity: sha512-Re10+NauLTMCudc7T5WLFLAwDhQ0JWdrMK+9B2M8zR5hRExKmsRDCBA7/aV/pNJFltmBFO5BAMlQFi/vq3nKOg==} engines: {node: '>=6'} @@ -11274,23 +10631,13 @@ packages: tar-fs@2.1.3: resolution: {integrity: sha512-090nwYJDmlhwFwEW3QQl+vaNnxsO2yVsd45eTKRBzSzu+hlb1w2K9inVq5b0ngXuLVqQ4ApvsUHHnu/zQNkWAg==} - tar-fs@3.0.9: - resolution: {integrity: sha512-XF4w9Xp+ZQgifKakjZYmFdkLoSWd34VGKcsTCwlNWM7QG3ZbaxnTsaBwnjFZqHRf/rROxaR8rXnbtwdvaDI+lA==} - tar-stream@2.2.0: resolution: {integrity: sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==} engines: {node: '>=6'} - tar-stream@3.1.7: - resolution: {integrity: sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ==} - tdigest@0.1.2: resolution: {integrity: sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==} - teeny-request@10.1.0: - resolution: {integrity: sha512-3ZnLvgWF29jikg1sAQ1g0o+lr5JX6sVgYvfUJazn7ZjJroDBUTWp44/+cFVX0bULjv4vci+rBD+oGVAkWqhUbw==} - engines: {node: '>=18'} - teeny-request@9.0.0: resolution: {integrity: sha512-resvxdc6Mgb7YEThw6G6bExlXKkv6+YbuzGg9xuXxSgxJF7Ozs+o8Y9+2R3sArdWdW8nOokoQb1yrpFB0pQK2g==} engines: {node: '>=14'} @@ -11334,9 +10681,6 @@ packages: resolution: {integrity: sha512-cAGWPIyOHU6zlmg88jwm7VRyXnMN7iV68OGAbYDk/Mh/xC/pzVPlQtY6ngoIH/5/tciuhGfvESU8GrHrcxD56w==} engines: {node: '>=8'} - text-decoder@1.2.3: - resolution: {integrity: sha512-3/o9z3X0X0fTupwsYvR03pJ/DjWuqqrfwBgTQzdWDiQSm9KitAyz/9WqsT2JQW7KV2m+bC2ol/zqpW37NHxLaA==} - text-hex@1.0.0: resolution: {integrity: sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg==} @@ -11392,8 +10736,8 @@ packages: tinyexec@0.3.2: resolution: {integrity: sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA==} - tinyglobby@0.2.14: - resolution: {integrity: sha512-tX5e7OM1HnYr2+a2C/4V0htOcSQcoSTH9KgJnVvNm5zm/cyEWKJ7j7YutsH9CxMdtOkkLFy2AHrMci9IM8IPZQ==} + tinyglobby@0.2.13: + resolution: {integrity: sha512-mEwzpUgrLySlveBwEVDMKk5B57bhLPYovRfPAXD5gA/98Opn0rCDj3GtLwFvCvH5RK9uPCExUROW5NjDwvqkxw==} engines: {node: '>=12.0.0'} tinyqueue@2.0.3: @@ -11458,6 +10802,9 @@ packages: resolution: {integrity: sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A==} hasBin: true + trim-lines@3.0.1: + resolution: {integrity: sha512-kRj8B+YHZCc9kQYdWfJB2/oUl9rA99qbowYYBtr4ui4mZyAQ2JpvVBd/6U2YloATfqBhBTSMhTpgBHtU0Mf3Rg==} + trim-newlines@3.0.1: resolution: {integrity: sha512-c1PTsA3tYrIsLGkJkzHF+w9F2EyxfXGo4UyJc4pFL++FMjnq0HJS69T3M7d//gKrFKwy429bouPescbjecU+Zw==} engines: {node: '>=8'} @@ -11507,9 +10854,6 @@ packages: engines: {node: '>=14.16'} hasBin: true - tsimportlib@0.0.5: - resolution: {integrity: sha512-qWQv/C3YB4Pwj77Z2HlORfy5EsWHcSYt66VQlMM0xZiKXwtoe1SxfpzmHX62sdJgzU6esrBGtyRIlx6O2OFPrQ==} - tslib@1.14.1: resolution: {integrity: sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==} @@ -11523,9 +10867,6 @@ packages: tunnel-agent@0.6.0: resolution: {integrity: sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==} - tweetnacl@1.0.3: - resolution: {integrity: sha512-6rt+RN7aOi1nGMyC4Xa5DdYiukl2UWCbcJft7YhxReBGQD7OAM8Pbxw6YMo4r2diNEA8FEmu32YOn9rhaiE5yw==} - type-check@0.4.0: resolution: {integrity: sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==} engines: {node: '>= 0.8.0'} @@ -11534,10 +10875,6 @@ packages: resolution: {integrity: sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==} engines: {node: '>=4'} - type-detect@4.1.0: - resolution: {integrity: sha512-Acylog8/luQ8L7il+geoSxhEkazvkslg7PSNKOX59mbB9cOveP5aq9h74Y7YU8yDpJwetzQQrfIwtf4Wp4LKcw==} - engines: {node: '>=4'} - type-fest@0.13.1: resolution: {integrity: sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg==} engines: {node: '>=10'} @@ -11626,6 +10963,10 @@ packages: uid2@0.0.4: resolution: {integrity: sha512-IevTus0SbGwQzYh3+fRsAMTVVPOoIVufzacXcHPmdlle1jUpq7BRL+mw3dgeLanvGZdwwbWhRV6XrcFNdBmjWA==} + uid2@1.0.0: + resolution: {integrity: sha512-+I6aJUv63YAcY9n4mQreLUt0d4lvwkkopDNmpomkAUz0fAkEMV9pRWxN0EjhW1YfRhcuyHg2v3mwddCDW1+LFQ==} + engines: {node: '>= 4.0.0'} + uid@2.0.2: resolution: {integrity: sha512-u3xV3X7uzvi5b1MncmZo3i2Aw222Zk1keqLA1YkHldREkAhAqi65wuPfe7lHx8H/Wzy+8CE7S7uS3jekIM5s8g==} engines: {node: '>=8'} @@ -11654,8 +10995,8 @@ packages: resolution: {integrity: sha512-lRfVq8fE8gz6QMBuDM6a+LO3IAzTi05H6gCVaUpir2E1Rwpo4ZUog45KpNXKC/Mn3Yb9UDuHumeFTo9iV/D9FQ==} engines: {node: '>=18'} - unified@10.1.2: - resolution: {integrity: sha512-pUSWAi/RAnVy1Pif2kAoeWNBa3JVrx0MId2LASj8G+7AiHWoKZNTomq6LG326T68U7/e263X6fTdcXIy7XnF7Q==} + unified@11.0.5: + resolution: {integrity: sha512-xKvGhPWw3k84Qjh8bI3ZeJjqnyadK+GEFtazSfZv/rKeTkTjOJho6mFqh2SM96iIcZokxiOpg78GazTSg8+KHA==} unique-random-array@1.0.1: resolution: {integrity: sha512-z9J/SV8CUIhIRROcHe9YUoAT6XthUJt0oUyLGgobiXJprDP9O9dsErNevvSaAv5BkhwFEVPn6nIEOKeNE6Ck1Q==} @@ -11673,23 +11014,23 @@ packages: resolution: {integrity: sha512-iQ1ZgWac3b8YxGThecQFRQiqgk6xFERRwHZIWeVVsqlbmgCRl0PY13R4mUkodNgctmg5b5odG1nyW/IbOxQTqg==} engines: {node: '>=6'} - unist-util-filter@4.0.1: - resolution: {integrity: sha512-RynicUM/vbOSTSiUK+BnaK9XMfmQUh6gyi7L6taNgc7FIf84GukXVV3ucGzEN/PhUUkdP5hb1MmXc+3cvPUm5Q==} + unist-util-filter@5.0.1: + resolution: {integrity: sha512-pHx7D4Zt6+TsfwylH9+lYhBhzyhEnCXs/lbq/Hstxno5z4gVdyc2WEW0asfjGKPyG4pEKrnBv5hdkO6+aRnQJw==} - unist-util-is@5.2.1: - resolution: {integrity: sha512-u9njyyfEh43npf1M+yGKDGVPbY/JWEemg5nH05ncKPfi+kBbKBJoTdsogMu33uhytuLlv9y0O7GH7fEdwLdLQw==} + unist-util-is@6.0.0: + resolution: {integrity: sha512-2qCTHimwdxLfz+YzdGfkqNlH0tLi9xjTnHddPmJwtIG9MGsdbutfTc4P+haPD7l7Cjxf/WZj+we5qfVPvvxfYw==} - unist-util-position@4.0.4: - resolution: {integrity: sha512-kUBE91efOWfIVBo8xzh/uZQ7p9ffYRtUbMRZBNFYwf0RK8koUMx6dGUfwylLOKmaT2cs4wSW96QoYUSXAyEtpg==} + unist-util-position@5.0.0: + resolution: {integrity: sha512-fucsC7HjXvkB5R3kTCO7kUjRdrS0BJt3M/FPxmHMBOm8JQi2BsHAHFsy27E0EolP8rp0NzXsJ+jNPyDWvOJZPA==} - unist-util-stringify-position@3.0.3: - resolution: {integrity: sha512-k5GzIBZ/QatR8N5X2y+drfpWG8IDBzdnVj6OInRNWm1oXrzydiaAT2OQiA8DPRRZyAKb9b6I2a6PxYklZD0gKg==} + unist-util-stringify-position@4.0.0: + resolution: {integrity: sha512-0ASV06AAoKCDkS2+xw5RXJywruurpbC4JZSm7nr7MOt1ojAzvyyaO+UxZf18j8FCF6kmzCZKcAgN/yu2gm2XgQ==} - unist-util-visit-parents@5.1.3: - resolution: {integrity: sha512-x6+y8g7wWMyQhL1iZfhIPhDAs7Xwbn9nRosDXl7qoPTSCy0yNxnKc+hWokFifWQIDGi154rdUqKvbCa4+1kLhg==} + unist-util-visit-parents@6.0.1: + resolution: {integrity: sha512-L/PqWzfTP9lzzEa6CKs0k2nARxTdZduw3zyh8d2NVBnsyvHjSX4TWse388YrrQKbvI8w20fGjGlhgT96WwKykw==} - unist-util-visit@4.1.2: - resolution: {integrity: sha512-MSd8OUGISqHdVvfY9TPhyK2VdUrPgxkUtWSuMHF6XAAFuL4LokseigBnZtPnJMu+FbynTkFNnFlyjxpVKujMRg==} + unist-util-visit@5.0.0: + resolution: {integrity: sha512-MR04uvD+07cwl/yhVuVWAtw+3GOR/knlL55Nd/wAdblk27GCVt3lqpTivy/tkJcZoNPzTwS1Y+KMojlLDhoTzg==} universal-cookie@4.0.4: resolution: {integrity: sha512-lbRVHoOMtItjWbM7TwDLdl8wug7izB0tq3/YVKhT/ahB4VDvWMyvnADfnJI8y6fSvsjh51Ix7lTGC6Tn4rMPhw==} @@ -11825,15 +11166,6 @@ packages: resolution: {integrity: sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A==} hasBin: true - uuid@3.4.0: - resolution: {integrity: sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==} - deprecated: Please upgrade to version 7 or higher. Older versions may use Math.random() in certain circumstances, which is known to be problematic. See https://v8.dev/blog/math-random for details. - hasBin: true - - uuid@7.0.3: - resolution: {integrity: sha512-DPSke0pXhTZgoF/d+WSt2QaKMCFSfx7QegxEWT+JOuHF5aWrKEn0G+ztjuJg/gG8/ItK+rbPCD/yNv8yyih6Cg==} - hasBin: true - uuid@8.3.2: resolution: {integrity: sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==} hasBin: true @@ -11887,14 +11219,14 @@ packages: resolution: {integrity: sha512-veufcmxri4e3XSrT0xwfUR7kguIkaxBeosDg00yDWhk49wdwkSUrvvsm7nc75e1PUyvIeZj6nS8VQRYz2/S4Xg==} engines: {node: '>=0.6.0'} - vfile-location@4.1.0: - resolution: {integrity: sha512-YF23YMyASIIJXpktBa4vIGLJ5Gs88UB/XePgqPmTa7cDA+JeO3yclbpheQYCHjVHBn/yePzrXuygIL+xbvRYHw==} + vfile-location@5.0.3: + resolution: {integrity: sha512-5yXvWDEgqeiYiBe1lbxYF7UMAIm/IcopxMHrMQDq3nvKcjPKIhZklUKL+AE7J7uApI4kwe2snsK+eI6UTj9EHg==} - vfile-message@3.1.4: - resolution: {integrity: sha512-fa0Z6P8HUrQN4BZaX05SIVXic+7kE3b05PWAtPuYP9QLHsLKYR7/AlLW3NtOrpXRLeawpDLMsVkmk5DG0NXgWw==} + vfile-message@4.0.2: + resolution: {integrity: sha512-jRDZ1IMLttGj41KcZvlrYAaI3CfqpLpfpf+Mfig13viT6NKvRzWZ+lXz0Y5D60w6uJIBAOGq9mSHf0gktF0duw==} - vfile@5.3.7: - resolution: {integrity: sha512-r7qlzkgErKjobAmyNIkkSpizsFPYiUPuJb5pNW1RB4JcYVZhs4lIbVqk8XPk033CV/1z8ss5pkax8SuhGpcG8g==} + vfile@6.0.3: + resolution: {integrity: sha512-KzIbH/9tXat2u30jf+smMwFCsno4wHVdNmzFyL+T/L3UGqqk6JKfVqOFOZEpZSHADH1k40ab6NUIXZq422ov3Q==} video-extensions@1.2.0: resolution: {integrity: sha512-TriMl18BHEsh2KuuSA065tbu4SNAC9fge7k8uKoTTofTq89+Xsg4K1BGbmSVETwUZhqSjd9KwRCNwXAW/buXMg==} @@ -11935,9 +11267,6 @@ packages: walker@1.0.8: resolution: {integrity: sha512-ts/8E8l5b7kY0vlWLewOkDXMmPdLcVV4GmOQLyxuSswIJsweeFZtAsMF7k1Nszz+TYBQrlYRmzOnr398y1JemQ==} - warning@2.1.0: - resolution: {integrity: sha512-O9pvum8nlCqIT5pRGo2WRQJPRG2bW/ZBeCzl7/8CWREjUW693juZpGup7zbRtuVcSKyGiRAIZLYsh3C0vq7FAg==} - warning@4.0.3: resolution: {integrity: sha512-rpJyN222KWIvHJ/F53XSZv0Zl/accqHR8et1kpaMTD/fLCRxtV8iX8czMzY7sVZupTI3zcUTg8eycS2kNF9l6w==} @@ -11957,10 +11286,6 @@ packages: web-namespaces@2.0.1: resolution: {integrity: sha512-bKr1DkiNa2krS7qxNtdrtHAmzuYGFQLiQ13TsorsdT6ULTkPLKuu5+GsFpDlg6JFjUTwX2DyhMPG2be8uPrqsQ==} - web-streams-polyfill@3.3.3: - resolution: {integrity: sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==} - engines: {node: '>= 8'} - web-streams-polyfill@4.0.0-beta.3: resolution: {integrity: sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==} engines: {node: '>= 14'} @@ -12030,9 +11355,6 @@ packages: resolution: {integrity: sha512-2HaXu1ytAso3qDk6TsW5PtkYJUWdY5SdXVPavu8wyTETHlNUUVb0eFXmIGsfupuZc94p7lCquNdTw4Zo6ITepg==} engines: {node: '>=0.16.0'} - webworkify@1.5.0: - resolution: {integrity: sha512-AMcUeyXAhbACL8S2hqqdqOLqvJ8ylmIbNwUIqQujRSouf4+eUFaXbG6F1Rbu+srlJMmxQWsiU7mOJi0nMBfM1g==} - whatwg-fetch@3.6.20: resolution: {integrity: sha512-EqhiFU6daOA8kpjOWTL0olhVOF3i7OrFzSYiGsEMB8GcXS+RrzauAERX65xMeNWVqxA6HXH2m69Z9LaKKdisfg==} @@ -12072,9 +11394,6 @@ packages: wordwrap@1.0.0: resolution: {integrity: sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q==} - workerpool@6.5.1: - resolution: {integrity: sha512-Fs4dNYcsdpYSAfVxhnl1L5zTksjvOJxtC5hzMNl+1t9B8hTJTdKDyZ5ju7ztgPy+ft9tBFXoOlDNiOT9WUXZlA==} - world-calendars@1.0.3: resolution: {integrity: sha512-sAjLZkBnsbHkHWVhrsCU5Sa/EVuf9QqgvrN8zyJ2L/F9FR9Oc6CvVK0674+PGAtmmmYQMH98tCUSO4QLQv3/TQ==} @@ -12109,6 +11428,18 @@ packages: utf-8-validate: optional: true + ws@8.17.1: + resolution: {integrity: sha512-6XQFvXTkbfUOZOKKILFG1PDK2NDQs4azKQl26T0YS5CxqWLgXajbPZ+h4gZekJyRqFU8pvnbAbbs/3TgRPy+GQ==} + engines: {node: '>=10.0.0'} + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: '>=5.0.2' + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + ws@8.18.2: resolution: {integrity: sha512-DMricUmwGZUVr++AEAe2uiVM7UoO9MAVZMDu05UQOaUII0lp+zOzLLU4Xqh/JvTqklB1T4uELaaPBKyjE1r4fQ==} engines: {node: '>=10.0.0'} @@ -12148,6 +11479,10 @@ packages: resolution: {integrity: sha512-yMqGBqtXyeN1e3TGYvgNgDVZ3j84W4cwkOXQswghol6APgZWaff9lnbvN7MHYJOiXsvGPXtjTYJEiC9J2wv9Eg==} engines: {node: '>=8.0'} + xmlhttprequest-ssl@2.1.2: + resolution: {integrity: sha512-TEU+nJVUUnA4CYJFLvK5X9AOeH4KvDvhIfm0vV1GaQRtchnG0hgK5p8hw/xjv8cunWYCsiPCSDzObPyhEwq3KQ==} + engines: {node: '>=0.4.0'} + xpath@0.0.32: resolution: {integrity: sha512-rxMJhSIoiO8vXcWvSifKqhvV96GjiD5wYb8/QHdoRyQvraTpp4IEv944nhGausZZ3u7dhQXteZuZbaqfpB7uYw==} engines: {node: '>=0.6.0'} @@ -12173,28 +11508,6 @@ packages: resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==} engines: {node: '>=0.4'} - xterm-addon-fit@0.6.0: - resolution: {integrity: sha512-9/7A+1KEjkFam0yxTaHfuk9LEvvTSBi0PZmEkzJqgafXPEXL9pCMAVV7rB09sX6ATRDXAdBpQhZkhKj7CGvYeg==} - deprecated: This package is now deprecated. Move to @xterm/addon-fit instead. - peerDependencies: - xterm: ^5.0.0 - - xterm-addon-web-links@0.7.0: - resolution: {integrity: sha512-6PqoqzzPwaeSq22skzbvyboDvSnYk5teUYEoKBwMYvhbkwOQkemZccjWHT5FnNA8o1aInTc4PRYAl4jjPucCKA==} - deprecated: This package is now deprecated. Move to @xterm/addon-web-links instead. - peerDependencies: - xterm: ^5.0.0 - - xterm-addon-webgl@0.13.0: - resolution: {integrity: sha512-xL4qBQWUHjFR620/8VHCtrTMVQsnZaAtd1IxFoiKPhC63wKp6b+73a45s97lb34yeo57PoqZhE9Jq5pB++ksPQ==} - deprecated: This package is now deprecated. Move to @xterm/addon-webgl instead. - peerDependencies: - xterm: ^5.0.0 - - xterm@5.0.0: - resolution: {integrity: sha512-tmVsKzZovAYNDIaUinfz+VDclraQpPUnAME+JawosgWRMphInDded/PuY0xmU5dOhyeYZsI0nz5yd8dPYsdLTA==} - deprecated: This package is now deprecated. Move to @xterm/xterm instead. - y-protocols@1.0.6: resolution: {integrity: sha512-vHRF2L6iT3rwj1jub/K5tYcTT/mEYDUppgNPXwp8fmLpui9f7Yeq3OEtTLVF012j39QnV+KEQpNqoN7CWU7Y9Q==} engines: {node: '>=16.0.0', npm: '>=8.0.0'} @@ -12228,10 +11541,6 @@ packages: resolution: {integrity: sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==} engines: {node: '>=12'} - yargs-unparser@2.0.0: - resolution: {integrity: sha512-7pRTIA9Qc1caZ0bZ6RYRGbHJthJWuakf+WmHK0rVeLkNrrGhfoabBNdue6kdINI6r4if7ocq9aD/n7xwKOdzOA==} - engines: {node: '>=10'} - yargs@16.2.0: resolution: {integrity: sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==} engines: {node: '>=10'} @@ -12252,9 +11561,9 @@ packages: resolution: {integrity: sha512-AyeEbWOu/TAXdxlV9wmGcR0+yh2j3vYPGOECcIj2S7MkrLyC7ne+oye2BKTItt0ii2PHk4cDy+95+LshzbXnGg==} engines: {node: '>=12.20'} - zeromq@5.3.1: - resolution: {integrity: sha512-4WDF9bNWWXe8OAI319bVw5dmG4BklEk8wzFGwRQxEzKb+0mgDU5J/jtyZPo0BEusVIU1+3mRQIEdT5LtQn+aAw==} - engines: {node: '>=6.0'} + zeromq@6.4.2: + resolution: {integrity: sha512-FnQlI4lEAewE4JexJ6kqQuBVzRf0Mg1n/qE3uXilfosf+X5lqJPiaYfdL/w4SzgAEVBTyqbMt9NbjwI5H89Yaw==} + engines: {node: '>= 12'} zlibjs@0.3.1: resolution: {integrity: sha512-+J9RrgTKOmlxFSDHo0pI1xM6BLVUv+o0ZT9ANtCxGkjIVCCUdx9alUF8Gm+dGLKbkkkidWIHFDZHDMpfITt4+w==} @@ -12275,6 +11584,10 @@ packages: zod@3.25.64: resolution: {integrity: sha512-hbP9FpSZf7pkS7hRVUrOjhwKJNyampPgtXKc3AN6DsWtoHsg2Sb4SQaS4Tcay380zSwd2VPo9G9180emBACp5g==} + zstd-napi@0.0.10: + resolution: {integrity: sha512-pwnG+auSiIrD2BNSIpPEUtcRSK33cfYmKo3sJPTohFiPqPci9F4SIRPR7gGeI45Maj4nFoyyxzT2YDxVXIIgzQ==} + engines: {node: ^12.22.0 || ^14.17.0 || ^15.12.0 || >=16} + zwitch@2.0.4: resolution: {integrity: sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A==} @@ -12282,7 +11595,7 @@ snapshots: '@ampproject/remapping@2.3.0': dependencies: - '@jridgewell/gen-mapping': 0.3.5 + '@jridgewell/gen-mapping': 0.3.8 '@jridgewell/trace-mapping': 0.3.25 '@ant-design/colors@6.0.0': @@ -12325,7 +11638,7 @@ snapshots: '@ant-design/cssinjs@1.23.0(react-dom@18.3.1(react@18.3.1))(react@18.3.1)': dependencies: - '@babel/runtime': 7.27.0 + '@babel/runtime': 7.27.1 '@emotion/hash': 0.8.0 '@emotion/unitless': 0.7.5 classnames: 2.5.1 @@ -12431,15 +11744,9 @@ snapshots: '@babel/code-frame@7.24.7': dependencies: - '@babel/highlight': 7.24.7 + '@babel/highlight': 7.25.7 picocolors: 1.1.0 - '@babel/code-frame@7.26.2': - dependencies: - '@babel/helper-validator-identifier': 7.25.9 - js-tokens: 4.0.0 - picocolors: 1.1.1 - '@babel/code-frame@7.27.1': dependencies: '@babel/helper-validator-identifier': 7.27.1 @@ -12451,156 +11758,78 @@ snapshots: '@babel/core@7.26.9': dependencies: '@ampproject/remapping': 2.3.0 - '@babel/code-frame': 7.26.2 - '@babel/generator': 7.27.0 + '@babel/code-frame': 7.27.1 + '@babel/generator': 7.27.1 '@babel/helper-compilation-targets': 7.27.0 - '@babel/helper-module-transforms': 7.26.0(@babel/core@7.26.9) + '@babel/helper-module-transforms': 7.27.1(@babel/core@7.26.9) '@babel/helpers': 7.27.0 - '@babel/parser': 7.27.0 - '@babel/template': 7.27.0 - '@babel/traverse': 7.27.0 - '@babel/types': 7.27.0 + '@babel/parser': 7.27.2 + '@babel/template': 7.27.2 + '@babel/traverse': 7.27.1 + '@babel/types': 7.27.1 convert-source-map: 2.0.0 - debug: 4.4.1(supports-color@8.1.1) + debug: 4.4.1 gensync: 1.0.0-beta.2 json5: 2.2.3 semver: 6.3.1 transitivePeerDependencies: - supports-color - '@babel/generator@7.27.0': - dependencies: - '@babel/parser': 7.27.0 - '@babel/types': 7.27.0 - '@jridgewell/gen-mapping': 0.3.8 - '@jridgewell/trace-mapping': 0.3.25 - jsesc: 3.1.0 - - '@babel/generator@7.27.5': + '@babel/generator@7.27.1': dependencies: - '@babel/parser': 7.27.5 - '@babel/types': 7.27.6 + '@babel/parser': 7.27.2 + '@babel/types': 7.27.1 '@jridgewell/gen-mapping': 0.3.8 '@jridgewell/trace-mapping': 0.3.25 jsesc: 3.1.0 - '@babel/helper-annotate-as-pure@7.27.3': - dependencies: - '@babel/types': 7.27.6 - '@babel/helper-compilation-targets@7.27.0': dependencies: '@babel/compat-data': 7.26.8 '@babel/helper-validator-option': 7.27.1 - browserslist: 4.24.4 + browserslist: 4.24.5 lru-cache: 5.1.1 semver: 6.3.1 - '@babel/helper-create-class-features-plugin@7.27.1(@babel/core@7.26.9)': - dependencies: - '@babel/core': 7.26.9 - '@babel/helper-annotate-as-pure': 7.27.3 - '@babel/helper-member-expression-to-functions': 7.27.1 - '@babel/helper-optimise-call-expression': 7.27.1 - '@babel/helper-replace-supers': 7.27.1(@babel/core@7.26.9) - '@babel/helper-skip-transparent-expression-wrappers': 7.27.1 - '@babel/traverse': 7.27.4 - semver: 6.3.1 - transitivePeerDependencies: - - supports-color - - '@babel/helper-member-expression-to-functions@7.27.1': - dependencies: - '@babel/traverse': 7.27.4 - '@babel/types': 7.27.6 - transitivePeerDependencies: - - supports-color - - '@babel/helper-module-imports@7.25.9': - dependencies: - '@babel/traverse': 7.27.0 - '@babel/types': 7.27.0 - transitivePeerDependencies: - - supports-color - '@babel/helper-module-imports@7.27.1': dependencies: - '@babel/traverse': 7.27.4 - '@babel/types': 7.27.6 - transitivePeerDependencies: - - supports-color - - '@babel/helper-module-transforms@7.26.0(@babel/core@7.26.9)': - dependencies: - '@babel/core': 7.26.9 - '@babel/helper-module-imports': 7.25.9 - '@babel/helper-validator-identifier': 7.25.9 - '@babel/traverse': 7.27.0 + '@babel/traverse': 7.27.1 + '@babel/types': 7.27.1 transitivePeerDependencies: - supports-color - '@babel/helper-module-transforms@7.27.3(@babel/core@7.26.9)': + '@babel/helper-module-transforms@7.27.1(@babel/core@7.26.9)': dependencies: '@babel/core': 7.26.9 '@babel/helper-module-imports': 7.27.1 '@babel/helper-validator-identifier': 7.27.1 - '@babel/traverse': 7.27.4 + '@babel/traverse': 7.27.1 transitivePeerDependencies: - supports-color - '@babel/helper-optimise-call-expression@7.27.1': - dependencies: - '@babel/types': 7.27.6 - '@babel/helper-plugin-utils@7.27.1': {} - '@babel/helper-replace-supers@7.27.1(@babel/core@7.26.9)': - dependencies: - '@babel/core': 7.26.9 - '@babel/helper-member-expression-to-functions': 7.27.1 - '@babel/helper-optimise-call-expression': 7.27.1 - '@babel/traverse': 7.27.4 - transitivePeerDependencies: - - supports-color - - '@babel/helper-skip-transparent-expression-wrappers@7.27.1': - dependencies: - '@babel/traverse': 7.27.4 - '@babel/types': 7.27.6 - transitivePeerDependencies: - - supports-color - - '@babel/helper-string-parser@7.25.9': {} - '@babel/helper-string-parser@7.27.1': {} - '@babel/helper-validator-identifier@7.24.7': {} - - '@babel/helper-validator-identifier@7.25.9': {} - '@babel/helper-validator-identifier@7.27.1': {} '@babel/helper-validator-option@7.27.1': {} '@babel/helpers@7.27.0': dependencies: - '@babel/template': 7.27.0 - '@babel/types': 7.27.0 + '@babel/template': 7.27.2 + '@babel/types': 7.27.1 - '@babel/highlight@7.24.7': + '@babel/highlight@7.25.7': dependencies: - '@babel/helper-validator-identifier': 7.24.7 + '@babel/helper-validator-identifier': 7.27.1 chalk: 2.4.2 js-tokens: 4.0.0 picocolors: 1.1.1 - '@babel/parser@7.27.0': - dependencies: - '@babel/types': 7.27.0 - - '@babel/parser@7.27.5': + '@babel/parser@7.27.2': dependencies: - '@babel/types': 7.27.6 + '@babel/types': 7.27.1 '@babel/plugin-syntax-async-generators@7.8.4(@babel/core@7.26.9)': dependencies: @@ -12667,46 +11896,11 @@ snapshots: '@babel/core': 7.26.9 '@babel/helper-plugin-utils': 7.27.1 - '@babel/plugin-syntax-typescript@7.25.9(@babel/core@7.26.9)': - dependencies: - '@babel/core': 7.26.9 - '@babel/helper-plugin-utils': 7.27.1 - '@babel/plugin-syntax-typescript@7.27.1(@babel/core@7.26.9)': dependencies: '@babel/core': 7.26.9 '@babel/helper-plugin-utils': 7.27.1 - '@babel/plugin-transform-modules-commonjs@7.27.1(@babel/core@7.26.9)': - dependencies: - '@babel/core': 7.26.9 - '@babel/helper-module-transforms': 7.27.3(@babel/core@7.26.9) - '@babel/helper-plugin-utils': 7.27.1 - transitivePeerDependencies: - - supports-color - - '@babel/plugin-transform-typescript@7.27.1(@babel/core@7.26.9)': - dependencies: - '@babel/core': 7.26.9 - '@babel/helper-annotate-as-pure': 7.27.3 - '@babel/helper-create-class-features-plugin': 7.27.1(@babel/core@7.26.9) - '@babel/helper-plugin-utils': 7.27.1 - '@babel/helper-skip-transparent-expression-wrappers': 7.27.1 - '@babel/plugin-syntax-typescript': 7.27.1(@babel/core@7.26.9) - transitivePeerDependencies: - - supports-color - - '@babel/preset-typescript@7.27.1(@babel/core@7.26.9)': - dependencies: - '@babel/core': 7.26.9 - '@babel/helper-plugin-utils': 7.27.1 - '@babel/helper-validator-option': 7.27.1 - '@babel/plugin-syntax-jsx': 7.27.1(@babel/core@7.26.9) - '@babel/plugin-transform-modules-commonjs': 7.27.1(@babel/core@7.26.9) - '@babel/plugin-transform-typescript': 7.27.1(@babel/core@7.26.9) - transitivePeerDependencies: - - supports-color - '@babel/runtime@7.25.6': dependencies: regenerator-runtime: 0.14.1 @@ -12715,50 +11909,29 @@ snapshots: dependencies: regenerator-runtime: 0.14.1 - '@babel/runtime@7.27.6': {} + '@babel/runtime@7.27.1': {} - '@babel/template@7.27.0': - dependencies: - '@babel/code-frame': 7.26.2 - '@babel/parser': 7.27.0 - '@babel/types': 7.27.0 + '@babel/runtime@7.27.6': {} '@babel/template@7.27.2': dependencies: '@babel/code-frame': 7.27.1 - '@babel/parser': 7.27.5 - '@babel/types': 7.27.6 - - '@babel/traverse@7.27.0': - dependencies: - '@babel/code-frame': 7.26.2 - '@babel/generator': 7.27.0 - '@babel/parser': 7.27.0 - '@babel/template': 7.27.0 - '@babel/types': 7.27.0 - debug: 4.4.1(supports-color@8.1.1) - globals: 11.12.0 - transitivePeerDependencies: - - supports-color + '@babel/parser': 7.27.2 + '@babel/types': 7.27.1 - '@babel/traverse@7.27.4': + '@babel/traverse@7.27.1': dependencies: '@babel/code-frame': 7.27.1 - '@babel/generator': 7.27.5 - '@babel/parser': 7.27.5 + '@babel/generator': 7.27.1 + '@babel/parser': 7.27.2 '@babel/template': 7.27.2 - '@babel/types': 7.27.6 - debug: 4.4.1(supports-color@8.1.1) + '@babel/types': 7.27.1 + debug: 4.4.1 globals: 11.12.0 transitivePeerDependencies: - supports-color - '@babel/types@7.27.0': - dependencies: - '@babel/helper-string-parser': 7.25.9 - '@babel/helper-validator-identifier': 7.25.9 - - '@babel/types@7.27.6': + '@babel/types@7.27.1': dependencies: '@babel/helper-string-parser': 7.27.1 '@babel/helper-validator-identifier': 7.27.1 @@ -12767,6 +11940,17 @@ snapshots: '@braintree/sanitize-url@7.1.1': {} + '@cfaester/enzyme-adapter-react-18@0.8.0(enzyme@3.11.0)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)': + dependencies: + enzyme: 3.11.0 + enzyme-shallow-equal: 1.0.7 + function.prototype.name: 1.1.8 + has: 1.0.4 + react: 18.3.1 + react-dom: 18.3.1(react@18.3.1) + react-is: 18.3.1 + react-shallow-renderer: 16.15.0(react@18.3.1) + '@cfworker/json-schema@4.1.1': {} '@chevrotain/cst-dts-gen@11.0.3': @@ -12795,7 +11979,7 @@ snapshots: awaiting: 3.0.0 cheerio: 1.0.0-rc.12 csv-parse: 5.6.0 - debug: 4.4.1(supports-color@8.1.1) + debug: 4.4.1 transitivePeerDependencies: - supports-color @@ -12809,11 +11993,19 @@ snapshots: '@cocalc/primus-responder@1.0.5': dependencies: - debug: 4.4.1(supports-color@8.1.1) + debug: 4.4.1 node-uuid: 1.4.8 transitivePeerDependencies: - supports-color + '@cocalc/redis-streams-adapter@0.2.3(socket.io-adapter@2.5.5)': + dependencies: + '@msgpack/msgpack': 2.8.0 + debug: 4.3.7 + socket.io-adapter: 2.5.5 + transitivePeerDependencies: + - supports-color + '@cocalc/widgets@1.2.0': {} '@cspell/cspell-bundled-dicts@8.19.4': @@ -12833,33 +12025,33 @@ snapshots: '@cspell/dict-docker': 1.1.14 '@cspell/dict-dotnet': 5.0.9 '@cspell/dict-elixir': 4.0.7 - '@cspell/dict-en-common-misspellings': 2.1.0 + '@cspell/dict-en-common-misspellings': 2.0.11 '@cspell/dict-en-gb': 1.1.33 - '@cspell/dict-en_us': 4.4.11 + '@cspell/dict-en_us': 4.4.9 '@cspell/dict-filetypes': 3.0.12 '@cspell/dict-flutter': 1.1.0 '@cspell/dict-fonts': 4.0.4 '@cspell/dict-fsharp': 1.1.0 '@cspell/dict-fullstack': 3.2.6 '@cspell/dict-gaming-terms': 1.1.1 - '@cspell/dict-git': 3.0.6 - '@cspell/dict-golang': 6.0.22 + '@cspell/dict-git': 3.0.5 + '@cspell/dict-golang': 6.0.21 '@cspell/dict-google': 1.0.8 '@cspell/dict-haskell': 4.0.5 '@cspell/dict-html': 4.0.11 '@cspell/dict-html-symbol-entities': 4.0.3 '@cspell/dict-java': 5.0.11 '@cspell/dict-julia': 1.1.0 - '@cspell/dict-k8s': 1.0.11 + '@cspell/dict-k8s': 1.0.10 '@cspell/dict-kotlin': 1.1.0 '@cspell/dict-latex': 4.0.3 '@cspell/dict-lorem-ipsum': 4.0.4 '@cspell/dict-lua': 4.0.7 '@cspell/dict-makefile': 1.0.4 - '@cspell/dict-markdown': 2.0.11(@cspell/dict-css@4.0.17)(@cspell/dict-html-symbol-entities@4.0.3)(@cspell/dict-html@4.0.11)(@cspell/dict-typescript@3.2.2) + '@cspell/dict-markdown': 2.0.10(@cspell/dict-css@4.0.17)(@cspell/dict-html-symbol-entities@4.0.3)(@cspell/dict-html@4.0.11)(@cspell/dict-typescript@3.2.2) '@cspell/dict-monkeyc': 1.0.10 '@cspell/dict-node': 5.0.7 - '@cspell/dict-npm': 5.2.6 + '@cspell/dict-npm': 5.2.4 '@cspell/dict-php': 4.0.14 '@cspell/dict-powershell': 5.0.14 '@cspell/dict-public-licenses': 2.0.13 @@ -12869,7 +12061,7 @@ snapshots: '@cspell/dict-rust': 4.0.11 '@cspell/dict-scala': 5.0.7 '@cspell/dict-shell': 1.1.0 - '@cspell/dict-software-terms': 5.1.0 + '@cspell/dict-software-terms': 5.0.10 '@cspell/dict-sql': 2.2.0 '@cspell/dict-svelte': 1.0.6 '@cspell/dict-swift': 2.0.5 @@ -12923,11 +12115,11 @@ snapshots: '@cspell/dict-elixir@4.0.7': {} - '@cspell/dict-en-common-misspellings@2.1.0': {} + '@cspell/dict-en-common-misspellings@2.0.11': {} '@cspell/dict-en-gb@1.1.33': {} - '@cspell/dict-en_us@4.4.11': {} + '@cspell/dict-en_us@4.4.9': {} '@cspell/dict-filetypes@3.0.12': {} @@ -12941,9 +12133,9 @@ snapshots: '@cspell/dict-gaming-terms@1.1.1': {} - '@cspell/dict-git@3.0.6': {} + '@cspell/dict-git@3.0.5': {} - '@cspell/dict-golang@6.0.22': {} + '@cspell/dict-golang@6.0.21': {} '@cspell/dict-google@1.0.8': {} @@ -12957,7 +12149,7 @@ snapshots: '@cspell/dict-julia@1.1.0': {} - '@cspell/dict-k8s@1.0.11': {} + '@cspell/dict-k8s@1.0.10': {} '@cspell/dict-kotlin@1.1.0': {} @@ -12969,7 +12161,7 @@ snapshots: '@cspell/dict-makefile@1.0.4': {} - '@cspell/dict-markdown@2.0.11(@cspell/dict-css@4.0.17)(@cspell/dict-html-symbol-entities@4.0.3)(@cspell/dict-html@4.0.11)(@cspell/dict-typescript@3.2.2)': + '@cspell/dict-markdown@2.0.10(@cspell/dict-css@4.0.17)(@cspell/dict-html-symbol-entities@4.0.3)(@cspell/dict-html@4.0.11)(@cspell/dict-typescript@3.2.2)': dependencies: '@cspell/dict-css': 4.0.17 '@cspell/dict-html': 4.0.11 @@ -12980,7 +12172,7 @@ snapshots: '@cspell/dict-node@5.0.7': {} - '@cspell/dict-npm@5.2.6': {} + '@cspell/dict-npm@5.2.4': {} '@cspell/dict-php@4.0.14': {} @@ -13002,7 +12194,7 @@ snapshots: '@cspell/dict-shell@1.1.0': {} - '@cspell/dict-software-terms@5.1.0': {} + '@cspell/dict-software-terms@5.0.10': {} '@cspell/dict-sql@2.2.0': {} @@ -13077,7 +12269,7 @@ snapshots: '@eslint/eslintrc@2.1.4': dependencies: ajv: 6.12.6 - debug: 4.4.1(supports-color@8.1.1) + debug: 4.4.1 espree: 9.6.1 globals: 13.24.0 ignore: 5.3.1 @@ -13128,12 +12320,6 @@ snapshots: optionalDependencies: typescript: 5.8.3 - '@google-ai/generativelanguage@3.2.0': - dependencies: - google-gax: 5.0.1-rc.0 - transitivePeerDependencies: - - supports-color - '@google-cloud/bigquery@7.9.4(encoding@0.1.13)': dependencies: '@google-cloud/common': 5.0.2(encoding@0.1.13) @@ -13238,7 +12424,7 @@ snapshots: '@humanwhocodes/config-array@0.13.0': dependencies: '@humanwhocodes/object-schema': 2.0.3 - debug: 4.4.1(supports-color@8.1.1) + debug: 4.4.1 minimatch: 3.1.2 transitivePeerDependencies: - supports-color @@ -13254,7 +12440,7 @@ snapshots: '@antfu/install-pkg': 1.0.0 '@antfu/utils': 8.1.1 '@iconify/types': 2.0.0 - debug: 4.4.1(supports-color@8.1.1) + debug: 4.4.1 globals: 15.15.0 kolorist: 1.8.0 local-pkg: 1.1.1 @@ -13266,6 +12452,8 @@ snapshots: dependencies: react: 18.3.1 + '@iovalkey/commands@0.1.0': {} + '@isaacs/cliui@8.0.2': dependencies: string-width: 5.1.2 @@ -13335,7 +12523,7 @@ snapshots: dependencies: '@jest/fake-timers': 29.7.0 '@jest/types': 29.6.3 - '@types/node': 24.0.1 + '@types/node': 18.19.111 jest-mock: 29.7.0 '@jest/expect-utils@29.7.0': @@ -13353,7 +12541,7 @@ snapshots: dependencies: '@jest/types': 29.6.3 '@sinonjs/fake-timers': 10.3.0 - '@types/node': 24.0.1 + '@types/node': 18.19.111 jest-message-util: 29.7.0 jest-mock: 29.7.0 jest-util: 29.7.0 @@ -13457,12 +12645,6 @@ snapshots: '@types/yargs': 17.0.24 chalk: 4.1.2 - '@jridgewell/gen-mapping@0.3.5': - dependencies: - '@jridgewell/set-array': 1.2.1 - '@jridgewell/sourcemap-codec': 1.5.0 - '@jridgewell/trace-mapping': 0.3.25 - '@jridgewell/gen-mapping@0.3.8': dependencies: '@jridgewell/set-array': 1.2.1 @@ -13643,7 +12825,7 @@ snapshots: camelcase: 6.3.0 decamelize: 1.2.0 js-tiktoken: 1.0.19 - langsmith: 0.3.31(openai@4.104.0(encoding@0.1.13)(ws@8.18.2)(zod@3.25.32)) + langsmith: 0.3.29(openai@4.104.0(encoding@0.1.13)(ws@8.18.2)(zod@3.25.32)) mustache: 4.2.0 p-queue: 6.6.2 p-retry: 4.6.2 @@ -13653,7 +12835,7 @@ snapshots: transitivePeerDependencies: - openai - '@langchain/google-genai@0.2.11(@langchain/core@0.3.58(openai@4.104.0(encoding@0.1.13)(ws@8.18.2)(zod@3.25.32)))': + '@langchain/google-genai@0.2.12(@langchain/core@0.3.58(openai@4.104.0(encoding@0.1.13)(ws@8.18.2)(zod@3.25.32)))': dependencies: '@google/generative-ai': 0.24.0 '@langchain/core': 0.3.58(openai@4.104.0(encoding@0.1.13)(ws@8.18.2)(zod@3.25.32)) @@ -13917,6 +13099,10 @@ snapshots: '@module-federation/runtime': 0.14.3 '@module-federation/sdk': 0.14.3 + '@msgpack/msgpack@2.8.0': {} + + '@msgpack/msgpack@3.1.2': {} + '@napi-rs/canvas-android-arm64@0.1.69': optional: true @@ -13963,38 +13149,6 @@ snapshots: '@napi-rs/triples@1.2.0': {} - '@nats-io/jetstream@3.0.0': - dependencies: - '@nats-io/nats-core': 3.0.0 - - '@nats-io/jwt@0.0.10-5': - dependencies: - '@nats-io/nkeys': 1.2.0-8 - - '@nats-io/kv@3.0.0': - dependencies: - '@nats-io/jetstream': 3.0.0 - '@nats-io/nats-core': 3.0.0 - - '@nats-io/nats-core@3.0.0': - dependencies: - '@nats-io/nkeys': 2.0.3 - '@nats-io/nuid': 2.0.3 - - '@nats-io/nkeys@1.2.0-8': - dependencies: - tweetnacl: 1.0.3 - - '@nats-io/nkeys@2.0.3': - dependencies: - tweetnacl: 1.0.3 - - '@nats-io/nuid@2.0.3': {} - - '@nats-io/services@3.0.0': - dependencies: - '@nats-io/nats-core': 3.0.0 - '@nestjs/axios@4.0.0(@nestjs/common@11.1.1(reflect-metadata@0.2.2)(rxjs@7.8.2))(axios@1.9.0)(rxjs@7.8.2)': dependencies: '@nestjs/common': 11.1.1(reflect-metadata@0.2.2)(rxjs@7.8.2) @@ -14068,7 +13222,7 @@ snapshots: '@types/xml2js': 0.4.14 '@xmldom/is-dom-node': 1.0.1 '@xmldom/xmldom': 0.8.10 - debug: 4.4.1(supports-color@8.1.1) + debug: 4.4.1 xml-crypto: 6.1.0 xml-encryption: 3.1.0 xml2js: 0.6.2 @@ -14367,7 +13521,7 @@ snapshots: '@rc-component/async-validator@5.0.4': dependencies: - '@babel/runtime': 7.27.0 + '@babel/runtime': 7.27.1 '@rc-component/color-picker@2.0.1(react-dom@18.3.1(react@18.3.1))(react@18.3.1)': dependencies: @@ -14387,7 +13541,7 @@ snapshots: '@rc-component/mini-decimal@1.1.0': dependencies: - '@babel/runtime': 7.27.0 + '@babel/runtime': 7.27.1 '@rc-component/mutate-observer@1.1.0(react-dom@18.3.1(react@18.3.1))(react@18.3.1)': dependencies: @@ -14399,7 +13553,7 @@ snapshots: '@rc-component/portal@1.1.2(react-dom@18.3.1(react@18.3.1))(react@18.3.1)': dependencies: - '@babel/runtime': 7.27.0 + '@babel/runtime': 7.27.1 classnames: 2.5.1 rc-util: 5.44.4(react-dom@18.3.1(react@18.3.1))(react@18.3.1) react: 18.3.1 @@ -14567,7 +13721,7 @@ snapshots: '@sendgrid/client@8.1.5': dependencies: '@sendgrid/helpers': 8.0.0 - axios: 1.9.0 + axios: 1.10.0 transitivePeerDependencies: - debug @@ -14586,10 +13740,6 @@ snapshots: '@sindresorhus/merge-streams@2.3.0': {} - '@sinonjs/commons@1.8.6': - dependencies: - type-detect: 4.0.8 - '@sinonjs/commons@3.0.1': dependencies: type-detect: 4.0.8 @@ -14598,24 +13748,25 @@ snapshots: dependencies: '@sinonjs/commons': 3.0.1 - '@sinonjs/formatio@2.0.0': + '@socket.io/cluster-adapter@0.2.2(socket.io-adapter@2.5.5)': dependencies: - samsam: 1.3.0 + debug: 4.3.7 + socket.io-adapter: 2.5.5 + transitivePeerDependencies: + - supports-color - '@sinonjs/formatio@3.2.2': - dependencies: - '@sinonjs/commons': 1.8.6 - '@sinonjs/samsam': 3.3.3 + '@socket.io/component-emitter@3.1.2': {} - '@sinonjs/samsam@3.3.3': + '@socket.io/redis-adapter@8.3.0(socket.io-adapter@2.5.5)': dependencies: - '@sinonjs/commons': 1.8.6 - array-from: 2.1.1 - lodash: 4.17.21 - - '@sinonjs/text-encoding@0.7.3': {} + debug: 4.3.7 + notepack.io: 3.0.1 + socket.io-adapter: 2.5.5 + uid2: 1.0.0 + transitivePeerDependencies: + - supports-color - '@speed-highlight/core@1.2.7': {} + '@socket.io/sticky@1.0.4': {} '@stripe/react-stripe-js@3.7.0(@stripe/stripe-js@5.10.0)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)': dependencies: @@ -14635,7 +13786,7 @@ snapshots: '@tokenizer/inflate@0.2.7': dependencies: - debug: 4.4.1(supports-color@8.1.1) + debug: 4.4.1 fflate: 0.8.2 token-types: 6.0.0 transitivePeerDependencies: @@ -14684,24 +13835,24 @@ snapshots: '@types/babel__core@7.20.5': dependencies: - '@babel/parser': 7.27.0 - '@babel/types': 7.27.6 + '@babel/parser': 7.27.2 + '@babel/types': 7.27.1 '@types/babel__generator': 7.6.8 '@types/babel__template': 7.4.4 '@types/babel__traverse': 7.20.5 '@types/babel__generator@7.6.8': dependencies: - '@babel/types': 7.27.6 + '@babel/types': 7.27.1 '@types/babel__template@7.4.4': dependencies: - '@babel/parser': 7.27.0 - '@babel/types': 7.27.6 + '@babel/parser': 7.27.2 + '@babel/types': 7.27.1 '@types/babel__traverse@7.20.5': dependencies: - '@babel/types': 7.27.6 + '@babel/types': 7.27.1 '@types/backbone@1.4.14': dependencies: @@ -14717,7 +13868,7 @@ snapshots: '@types/better-sqlite3@7.6.13': dependencies: - '@types/node': 24.0.1 + '@types/node': 18.19.111 '@types/body-parser@1.19.5': dependencies: @@ -14754,6 +13905,10 @@ snapshots: '@types/cookie@0.6.0': {} + '@types/cors@2.8.18': + dependencies: + '@types/node': 18.19.111 + '@types/d3-array@3.2.1': {} '@types/d3-axis@3.0.6': @@ -14873,7 +14028,7 @@ snapshots: '@types/debug@4.1.12': dependencies: - '@types/ms': 2.1.0 + '@types/ms': 0.7.31 '@types/dot-object@2.1.6': {} @@ -14944,9 +14099,13 @@ snapshots: dependencies: '@types/unist': 2.0.11 + '@types/hast@3.0.4': + dependencies: + '@types/unist': 3.0.3 + '@types/hoist-non-react-statics@3.3.1': dependencies: - '@types/react': 18.3.10 + '@types/react': 18.3.23 hoist-non-react-statics: 3.3.2 '@types/html-minifier-terser@6.1.0': {} @@ -14992,20 +14151,12 @@ snapshots: dependencies: '@types/node': 18.19.111 - '@types/ldapjs@2.2.5': - dependencies: - '@types/node': 18.19.111 - '@types/linkify-it@5.0.0': {} '@types/lodash@4.17.17': {} '@types/long@4.0.2': {} - '@types/long@5.0.0': - dependencies: - long: 5.3.1 - '@types/mapbox__point-geometry@0.1.4': {} '@types/mapbox__vector-tile@1.3.4': @@ -15021,6 +14172,10 @@ snapshots: '@types/md5@2.3.5': {} + '@types/mdast@4.0.4': + dependencies: + '@types/unist': 3.0.3 + '@types/mdurl@2.0.0': {} '@types/mime@1.3.5': {} @@ -15029,12 +14184,10 @@ snapshots: '@types/minimist@1.2.2': {} - '@types/mocha@10.0.10': {} + '@types/ms@0.7.31': {} '@types/ms@0.7.34': {} - '@types/ms@2.1.0': {} - '@types/node-cleanup@2.1.5': {} '@types/node-fetch@2.6.12': @@ -15058,8 +14211,6 @@ snapshots: dependencies: undici-types: 7.8.0 - '@types/node@9.6.61': {} - '@types/nodemailer@6.4.17': dependencies: '@types/node': 18.19.111 @@ -15070,8 +14221,6 @@ snapshots: dependencies: '@types/node': 18.19.111 - '@types/parse5@6.0.3': {} - '@types/passport-google-oauth20@2.0.16': dependencies: '@types/express': 4.17.23 @@ -15101,8 +14250,6 @@ snapshots: pg-protocol: 1.8.0 pg-types: 2.2.0 - '@types/pica@5.1.3': {} - '@types/primus@7.3.9': dependencies: '@types/node': 18.19.111 @@ -15128,11 +14275,6 @@ snapshots: hoist-non-react-statics: 3.3.2 redux: 4.2.1 - '@types/react@18.3.10': - dependencies: - '@types/prop-types': 15.7.13 - csstype: 3.1.3 - '@types/react@18.3.23': dependencies: '@types/prop-types': 15.7.13 @@ -15212,17 +14354,14 @@ snapshots: '@types/unist@2.0.11': {} + '@types/unist@3.0.3': {} + '@types/use-sync-external-store@0.0.3': {} '@types/uuid@10.0.0': {} '@types/uuid@8.3.4': {} - '@types/watchpack@2.4.4': - dependencies: - '@types/graceful-fs': 4.1.9 - '@types/node': 18.19.111 - '@types/ws@8.18.1': dependencies: '@types/node': 18.19.111 @@ -15253,7 +14392,7 @@ snapshots: '@typescript-eslint/type-utils': 6.21.0(eslint@8.57.1)(typescript@5.8.3) '@typescript-eslint/utils': 6.21.0(eslint@8.57.1)(typescript@5.8.3) '@typescript-eslint/visitor-keys': 6.21.0 - debug: 4.4.1(supports-color@8.1.1) + debug: 4.4.1 eslint: 8.57.1 graphemer: 1.4.0 ignore: 5.3.1 @@ -15271,7 +14410,7 @@ snapshots: '@typescript-eslint/types': 6.21.0 '@typescript-eslint/typescript-estree': 6.21.0(typescript@5.8.3) '@typescript-eslint/visitor-keys': 6.21.0 - debug: 4.4.1(supports-color@8.1.1) + debug: 4.4.1 eslint: 8.57.1 optionalDependencies: typescript: 5.8.3 @@ -15287,7 +14426,7 @@ snapshots: dependencies: '@typescript-eslint/typescript-estree': 6.21.0(typescript@5.8.3) '@typescript-eslint/utils': 6.21.0(eslint@8.57.1)(typescript@5.8.3) - debug: 4.4.1(supports-color@8.1.1) + debug: 4.4.1 eslint: 8.57.1 ts-api-utils: 1.4.3(typescript@5.8.3) optionalDependencies: @@ -15301,7 +14440,7 @@ snapshots: dependencies: '@typescript-eslint/types': 6.21.0 '@typescript-eslint/visitor-keys': 6.21.0 - debug: 4.4.1(supports-color@8.1.1) + debug: 4.4.1 globby: 11.1.0 is-glob: 4.0.3 minimatch: 9.0.3 @@ -15331,13 +14470,13 @@ snapshots: '@typescript-eslint/types': 6.21.0 eslint-visitor-keys: 3.4.3 - '@uiw/react-textarea-code-editor@2.1.9(@babel/runtime@7.27.6)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)': + '@uiw/react-textarea-code-editor@3.1.1(@babel/runtime@7.27.6)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)': dependencies: '@babel/runtime': 7.27.6 react: 18.3.1 react-dom: 18.3.1(react@18.3.1) - rehype: 12.0.1 - rehype-prism-plus: 1.6.3 + rehype: 13.0.2 + rehype-prism-plus: 2.0.0 '@ungap/structured-clone@1.3.0': {} @@ -15432,6 +14571,20 @@ snapshots: '@xmldom/xmldom@0.9.8': {} + '@xterm/addon-fit@0.10.0(@xterm/xterm@5.5.0)': + dependencies: + '@xterm/xterm': 5.5.0 + + '@xterm/addon-web-links@0.11.0(@xterm/xterm@5.5.0)': + dependencies: + '@xterm/xterm': 5.5.0 + + '@xterm/addon-webgl@0.18.0(@xterm/xterm@5.5.0)': + dependencies: + '@xterm/xterm': 5.5.0 + + '@xterm/xterm@5.5.0': {} + '@xtuc/ieee754@1.2.0': {} '@xtuc/long@4.2.2': {} @@ -15497,7 +14650,7 @@ snapshots: agent-base@6.0.2: dependencies: - debug: 4.4.1(supports-color@8.1.1) + debug: 4.4.1 transitivePeerDependencies: - supports-color @@ -15538,8 +14691,6 @@ snapshots: anser@2.3.2: {} - ansi-colors@4.1.3: {} - ansi-escapes@4.3.2: dependencies: type-fest: 0.21.3 @@ -15650,8 +14801,6 @@ snapshots: array-flatten@1.1.1: {} - array-from@2.1.1: {} - array-includes@3.1.8: dependencies: call-bind: 1.0.8 @@ -15681,6 +14830,15 @@ snapshots: array-uniq@1.0.3: {} + array.prototype.filter@1.0.4: + dependencies: + call-bind: 1.0.8 + define-properties: 1.2.1 + es-abstract: 1.23.9 + es-array-method-boxes-properly: 1.0.0 + es-object-atoms: 1.1.1 + is-string: 1.1.1 + array.prototype.findlast@1.2.5: dependencies: call-bind: 1.0.8 @@ -15728,6 +14886,10 @@ snapshots: asap@2.0.6: {} + ascii-table3@1.0.1: + dependencies: + printable-characters: 1.0.42 + asn1@0.2.6: dependencies: safer-buffer: 2.1.2 @@ -15786,7 +14948,7 @@ snapshots: awaiting@3.0.0: {} - axios@1.9.0: + axios@1.10.0: dependencies: follow-redirects: 1.15.9(debug@4.4.1) form-data: 4.0.2 @@ -15794,7 +14956,13 @@ snapshots: transitivePeerDependencies: - debug - b4a@1.6.7: {} + axios@1.9.0: + dependencies: + follow-redirects: 1.15.9(debug@4.4.1) + form-data: 4.0.2 + proxy-from-env: 1.1.0 + transitivePeerDependencies: + - debug babel-jest@29.7.0(@babel/core@7.26.9): dependencies: @@ -15829,7 +14997,7 @@ snapshots: babel-plugin-jest-hoist@29.6.3: dependencies: '@babel/template': 7.27.2 - '@babel/types': 7.27.6 + '@babel/types': 7.27.1 '@types/babel__core': 7.20.5 '@types/babel__traverse': 7.20.5 @@ -15880,31 +15048,6 @@ snapshots: balanced-match@1.0.2: {} - bare-events@2.5.4: - optional: true - - bare-fs@4.1.2: - dependencies: - bare-events: 2.5.4 - bare-path: 3.0.0 - bare-stream: 2.6.5(bare-events@2.5.4) - optional: true - - bare-os@3.6.1: - optional: true - - bare-path@3.0.0: - dependencies: - bare-os: 3.6.1 - optional: true - - bare-stream@2.6.5(bare-events@2.5.4): - dependencies: - streamx: 2.22.0 - optionalDependencies: - bare-events: 2.5.4 - optional: true - base-64@1.0.0: {} base16@1.0.0: {} @@ -15915,6 +15058,8 @@ snapshots: base64-js@1.5.1: {} + base64id@2.0.0: {} + base64url@3.0.1: {} basic-auth@2.0.1: @@ -15925,8 +15070,6 @@ snapshots: batch@0.6.1: {} - bcryptjs@2.4.3: {} - better-sqlite3@11.10.0: dependencies: bindings: 1.5.0 @@ -15999,8 +15142,6 @@ snapshots: bootstrap@3.4.1: {} - bottleneck@2.19.5: {} - brace-expansion@1.1.11: dependencies: balanced-match: 1.0.2 @@ -16014,14 +15155,12 @@ snapshots: dependencies: fill-range: 7.1.1 - browser-stdout@1.3.1: {} - - browserslist@4.24.4: + browserslist@4.24.5: dependencies: - caniuse-lite: 1.0.30001713 - electron-to-chromium: 1.5.136 + caniuse-lite: 1.0.30001718 + electron-to-chromium: 1.5.157 node-releases: 2.0.19 - update-browserslist-db: 1.1.3(browserslist@4.24.4) + update-browserslist-db: 1.1.3(browserslist@4.24.5) browserslist@4.25.0: dependencies: @@ -16119,6 +15258,8 @@ snapshots: caniuse-lite@1.0.30001713: {} + caniuse-lite@1.0.30001718: {} + caniuse-lite@1.0.30001723: {} canvas-fit@1.5.0: @@ -16211,7 +15352,7 @@ snapshots: domhandler: 5.0.3 domutils: 3.2.2 htmlparser2: 8.0.2 - parse5: 7.3.0 + parse5: 7.2.1 parse5-htmlparser2-tree-adapter: 7.1.0 chevrotain-allstar@0.3.1(chevrotain@11.0.3): @@ -16317,6 +15458,10 @@ snapshots: clsx@1.2.1: {} + cluster-key-slot@1.1.2: {} + + cmake-ts@0.6.1: {} + co@4.6.0: {} codemirror@5.65.19: {} @@ -16325,14 +15470,6 @@ snapshots: dependencies: mkpath: 0.1.0 - coffee-coverage@3.0.1: - dependencies: - argparse: 1.0.10 - coffeescript: 2.7.0 - lodash: 4.17.21 - minimatch: 3.1.2 - pkginfo: 0.4.1 - coffee-loader@3.0.0(coffeescript@2.7.0)(webpack@5.99.5): dependencies: coffeescript: 2.7.0 @@ -16340,8 +15477,6 @@ snapshots: coffee-react-transform@4.0.0: {} - coffee-script@1.12.7: {} - coffeescript@2.7.0: {} collect-v8-coverage@1.0.2: {} @@ -16403,11 +15538,6 @@ snapshots: color-convert: 1.9.3 color-string: 1.9.1 - color@4.2.3: - dependencies: - color-convert: 2.0.1 - color-string: 1.9.1 - colorette@2.0.20: {} colornames@1.1.1: {} @@ -16510,11 +15640,13 @@ snapshots: connected@0.0.2: {} + consistent-hash@1.2.2: {} + consola@2.15.3: {} consola@3.4.2: {} - console-table-printer@2.14.3: + console-table-printer@2.12.1: dependencies: simple-wcswidth: 1.0.1 @@ -16725,9 +15857,7 @@ snapshots: fast-json-stable-stringify: 2.1.0 file-entry-cache: 9.1.0 semver: 7.7.1 - tinyglobby: 0.2.14 - - css-color-names@0.0.4: {} + tinyglobby: 0.2.13 css-font-size-keywords@1.0.0: {} @@ -17032,8 +16162,6 @@ snapshots: dependencies: malevic: 0.20.2 - data-uri-to-buffer@4.0.1: {} - data-uri-to-buffer@6.0.2: {} data-view-buffer@1.0.2: @@ -17070,17 +16198,13 @@ snapshots: dependencies: ms: 2.1.2 - debug@4.4.1(supports-color@8.1.1): + debug@4.3.7: dependencies: ms: 2.1.3 - optionalDependencies: - supports-color: 8.1.1 - debug@4.4.1(supports-color@9.4.0): + debug@4.4.1: dependencies: ms: 2.1.3 - optionalDependencies: - supports-color: 9.4.0 decamelize-keys@1.1.0: dependencies: @@ -17089,8 +16213,6 @@ snapshots: decamelize@1.2.0: {} - decamelize@4.0.0: {} - decimal.js-light@2.5.1: {} decimal.js@10.4.3: {} @@ -17105,15 +16227,6 @@ snapshots: dedent@1.5.3: {} - deep-equal@1.1.2: - dependencies: - is-arguments: 1.2.0 - is-date-object: 1.1.0 - is-regex: 1.2.1 - object-is: 1.1.6 - object-keys: 1.1.1 - regexp.prototype.flags: 1.5.4 - deep-extend@0.6.0: {} deep-is@0.1.4: {} @@ -17169,6 +16282,8 @@ snapshots: delayed-stream@1.0.0: {} + denque@2.1.0: {} + depd@1.1.2: {} depd@2.0.0: {} @@ -17188,6 +16303,10 @@ snapshots: detect-node@2.1.0: {} + devlop@1.1.0: + dependencies: + dequal: 2.0.3 + dezalgo@1.0.4: dependencies: asap: 2.0.6 @@ -17210,16 +16329,14 @@ snapshots: diff-sequences@29.6.3: {} - diff@3.5.0: {} - - diff@5.2.0: {} - dir-glob@3.0.1: dependencies: path-type: 4.0.0 direction@1.0.4: {} + discontinuous-range@1.0.0: {} + diskusage@1.2.0: dependencies: es6-promise: 4.2.8 @@ -17367,7 +16484,7 @@ snapshots: dependencies: jake: 10.9.2 - electron-to-chromium@1.5.136: {} + electron-to-chromium@1.5.157: {} electron-to-chromium@1.5.167: {} @@ -17395,13 +16512,6 @@ snapshots: enabled@2.0.0: {} - enchannel-zmq-backend@9.1.23(rxjs@7.8.2): - dependencies: - '@nteract/messaging': 7.0.20 - jmp: 2.0.0 - rxjs: 7.8.2 - uuid: 7.0.3 - encodeurl@1.0.2: {} encodeurl@2.0.0: {} @@ -17414,6 +16524,36 @@ snapshots: dependencies: once: 1.4.0 + engine.io-client@6.6.3: + dependencies: + '@socket.io/component-emitter': 3.1.2 + debug: 4.3.7 + engine.io-parser: 5.2.3 + ws: 8.17.1 + xmlhttprequest-ssl: 2.1.2 + transitivePeerDependencies: + - bufferutil + - supports-color + - utf-8-validate + + engine.io-parser@5.2.3: {} + + engine.io@6.6.4: + dependencies: + '@types/cors': 2.8.18 + '@types/node': 18.19.111 + accepts: 1.3.8 + base64id: 2.0.0 + cookie: 0.7.2 + cors: 2.8.5 + debug: 4.3.7 + engine.io-parser: 5.2.3 + ws: 8.17.1 + transitivePeerDependencies: + - bufferutil + - supports-color + - utf-8-validate + enhanced-resolve@5.18.1: dependencies: graceful-fs: 4.2.11 @@ -17431,6 +16571,36 @@ snapshots: env-variable@0.0.6: {} + enzyme-shallow-equal@1.0.7: + dependencies: + hasown: 2.0.2 + object-is: 1.1.6 + + enzyme@3.11.0: + dependencies: + array.prototype.flat: 1.3.3 + cheerio: 1.0.0-rc.10 + enzyme-shallow-equal: 1.0.7 + function.prototype.name: 1.1.8 + has: 1.0.4 + html-element-map: 1.3.1 + is-boolean-object: 1.2.2 + is-callable: 1.2.7 + is-number-object: 1.1.1 + is-regex: 1.2.1 + is-string: 1.1.1 + is-subset: 0.1.1 + lodash.escape: 4.0.1 + lodash.isequal: 4.5.0 + object-inspect: 1.13.4 + object-is: 1.1.6 + object.assign: 4.1.7 + object.entries: 1.1.9 + object.values: 1.2.1 + raf: 3.4.1 + rst-selector-parser: 2.2.3 + string.prototype.trim: 1.2.10 + errno@0.1.8: dependencies: prr: 1.0.1 @@ -17498,6 +16668,8 @@ snapshots: unbox-primitive: 1.1.0 which-typed-array: 1.1.19 + es-array-method-boxes-properly@1.0.0: {} + es-class@2.1.1: {} es-define-property@1.0.0: @@ -17675,7 +16847,7 @@ snapshots: ajv: 6.12.6 chalk: 4.1.2 cross-spawn: 7.0.3 - debug: 4.4.1(supports-color@8.1.1) + debug: 4.4.1 doctrine: 3.0.0 escape-string-regexp: 4.0.0 eslint-scope: 7.2.2 @@ -17761,18 +16933,6 @@ snapshots: signal-exit: 3.0.7 strip-final-newline: 2.0.0 - execa@8.0.1: - dependencies: - cross-spawn: 7.0.3 - get-stream: 8.0.1 - human-signals: 5.0.0 - is-stream: 3.0.0 - merge-stream: 2.0.0 - npm-run-path: 5.3.0 - onetime: 6.0.0 - signal-exit: 4.1.0 - strip-final-newline: 3.0.0 - exit-hook@4.0.0: {} exit@0.1.2: {} @@ -17878,8 +17038,6 @@ snapshots: fast-equals@5.2.2: {} - fast-fifo@1.3.2: {} - fast-glob@3.3.2: dependencies: '@nodelib/fs.stat': 2.0.5 @@ -17920,17 +17078,10 @@ snapshots: dependencies: bser: 2.1.1 - fdir@6.4.6(picomatch@4.0.2): + fdir@6.4.4(picomatch@4.0.2): optionalDependencies: picomatch: 4.0.2 - fetch-blob@3.2.0: - dependencies: - node-domexception: 1.0.0 - web-streams-polyfill: 3.3.3 - - fflate@0.7.3: {} - fflate@0.8.2: {} figures@3.2.0: @@ -17948,7 +17099,7 @@ snapshots: file-type@20.5.0: dependencies: '@tokenizer/inflate': 0.2.7 - strtok3: 10.3.1 + strtok3: 10.2.2 token-types: 6.0.0 uint8array-extras: 1.4.0 transitivePeerDependencies: @@ -18011,8 +17162,6 @@ snapshots: flat-zip@1.0.1: {} - flat@5.0.2: {} - flatted@3.3.3: {} flatten-vertex-data@1.0.2: @@ -18021,7 +17170,7 @@ snapshots: follow-redirects@1.15.9(debug@4.4.1): optionalDependencies: - debug: 4.4.1(supports-color@8.1.1) + debug: 4.4.1 font-atlas@2.1.0: dependencies: @@ -18062,10 +17211,6 @@ snapshots: node-domexception: 1.0.0 web-streams-polyfill: 4.0.0-beta.3 - formdata-polyfill@4.0.10: - dependencies: - fetch-blob: 3.2.0 - formidable@3.5.2: dependencies: dezalgo: 1.0.4 @@ -18134,14 +17279,6 @@ snapshots: - encoding - supports-color - gaxios@7.0.0-rc.4: - dependencies: - extend: 3.0.2 - https-proxy-agent: 7.0.6 - node-fetch: 3.3.2 - transitivePeerDependencies: - - supports-color - gcp-metadata@6.1.1(encoding@0.1.13): dependencies: gaxios: 6.7.1(encoding@0.1.13) @@ -18151,14 +17288,6 @@ snapshots: - encoding - supports-color - gcp-metadata@7.0.0-rc.1: - dependencies: - gaxios: 7.0.0-rc.4 - google-logging-utils: 1.1.1 - json-bigint: 1.0.0 - transitivePeerDependencies: - - supports-color - gensequence@7.0.0: {} gensync@1.0.0-beta.2: {} @@ -18222,8 +17351,6 @@ snapshots: get-stream@6.0.1: {} - get-stream@8.0.1: {} - get-symbol-description@1.1.0: dependencies: call-bound: 1.0.4 @@ -18234,7 +17361,7 @@ snapshots: dependencies: basic-ftp: 5.0.5 data-uri-to-buffer: 6.0.2 - debug: 4.4.1(supports-color@8.1.1) + debug: 4.4.1 transitivePeerDependencies: - supports-color @@ -18311,14 +17438,6 @@ snapshots: once: 1.4.0 path-is-absolute: 1.0.1 - glob@8.1.0: - dependencies: - fs.realpath: 1.0.0 - inflight: 1.0.6 - inherits: 2.0.4 - minimatch: 5.1.6 - once: 1.4.0 - glob@9.3.5: dependencies: fs.realpath: 1.0.0 @@ -18442,7 +17561,7 @@ snapshots: graceful-fs: 4.2.11 inherits: 2.0.4 map-limit: 0.0.1 - resolve: 1.22.8 + resolve: 1.22.10 glslify@7.1.1: dependencies: @@ -18456,25 +17575,12 @@ snapshots: glslify-bundle: 5.1.1 glslify-deps: 1.3.2 minimist: 1.2.8 - resolve: 1.22.8 + resolve: 1.22.10 stack-trace: 0.0.9 static-eval: 2.1.1 through2: 2.0.5 xtend: 4.0.2 - glur@1.1.2: {} - - google-auth-library@10.0.0-rc.1: - dependencies: - base64-js: 1.5.1 - ecdsa-sig-formatter: 1.0.11 - gaxios: 7.0.0-rc.4 - gcp-metadata: 7.0.0-rc.1 - gtoken: 8.0.0-rc.1 - jws: 4.0.0 - transitivePeerDependencies: - - supports-color - google-auth-library@9.15.1(encoding@0.1.13): dependencies: base64-js: 1.5.1 @@ -18505,27 +17611,8 @@ snapshots: - encoding - supports-color - google-gax@5.0.1-rc.0: - dependencies: - '@grpc/grpc-js': 1.13.3 - '@grpc/proto-loader': 0.7.13 - '@types/long': 5.0.0 - abort-controller: 3.0.0 - duplexify: 4.1.3 - google-auth-library: 10.0.0-rc.1 - google-logging-utils: 1.1.1 - node-fetch: 3.3.2 - object-hash: 3.0.0 - proto3-json-serializer: 3.0.0 - protobufjs: 7.4.0 - retry-request: 8.0.0 - transitivePeerDependencies: - - supports-color - google-logging-utils@0.0.2: {} - google-logging-utils@1.1.1: {} - googleapis-common@7.2.0(encoding@0.1.13): dependencies: extend: 3.0.2 @@ -18546,8 +17633,6 @@ snapshots: - encoding - supports-color - googlediff@0.1.0: {} - gopd@1.2.0: {} got@6.7.1: @@ -18584,13 +17669,6 @@ snapshots: - encoding - supports-color - gtoken@8.0.0-rc.1: - dependencies: - gaxios: 7.0.0-rc.4 - jws: 4.0.0 - transitivePeerDependencies: - - supports-color - gzip-size@6.0.0: dependencies: duplexer: 0.1.2 @@ -18652,66 +17730,61 @@ snapshots: dependencies: has-symbols: 1.1.0 + has@1.0.4: {} + hasown@2.0.2: dependencies: function-bind: 1.1.2 - hast-util-from-parse5@7.1.2: + hast-util-from-html@2.0.3: dependencies: - '@types/hast': 2.3.10 - '@types/unist': 2.0.11 - hastscript: 7.2.0 - property-information: 6.5.0 - vfile: 5.3.7 - vfile-location: 4.1.0 + '@types/hast': 3.0.4 + devlop: 1.1.0 + hast-util-from-parse5: 8.0.3 + parse5: 7.3.0 + vfile: 6.0.3 + vfile-message: 4.0.2 + + hast-util-from-parse5@8.0.3: + dependencies: + '@types/hast': 3.0.4 + '@types/unist': 3.0.3 + devlop: 1.1.0 + hastscript: 9.0.1 + property-information: 7.1.0 + vfile: 6.0.3 + vfile-location: 5.0.3 web-namespaces: 2.0.1 hast-util-parse-selector@3.1.1: dependencies: '@types/hast': 2.3.10 - hast-util-raw@7.2.3: + hast-util-parse-selector@4.0.0: dependencies: - '@types/hast': 2.3.10 - '@types/parse5': 6.0.3 - hast-util-from-parse5: 7.1.2 - hast-util-to-parse5: 7.1.0 - html-void-elements: 2.0.1 - parse5: 6.0.1 - unist-util-position: 4.0.4 - unist-util-visit: 4.1.2 - vfile: 5.3.7 - web-namespaces: 2.0.1 - zwitch: 2.0.4 + '@types/hast': 3.0.4 - hast-util-to-html@8.0.4: + hast-util-to-html@9.0.5: dependencies: - '@types/hast': 2.3.10 - '@types/unist': 2.0.11 + '@types/hast': 3.0.4 + '@types/unist': 3.0.3 ccount: 2.0.1 comma-separated-tokens: 2.0.3 - hast-util-raw: 7.2.3 - hast-util-whitespace: 2.0.1 - html-void-elements: 2.0.1 - property-information: 6.5.0 + hast-util-whitespace: 3.0.0 + html-void-elements: 3.0.0 + mdast-util-to-hast: 13.2.0 + property-information: 7.1.0 space-separated-tokens: 2.0.2 stringify-entities: 4.0.4 zwitch: 2.0.4 - hast-util-to-parse5@7.1.0: + hast-util-to-string@3.0.1: dependencies: - '@types/hast': 2.3.10 - comma-separated-tokens: 2.0.3 - property-information: 6.5.0 - space-separated-tokens: 2.0.2 - web-namespaces: 2.0.1 - zwitch: 2.0.4 + '@types/hast': 3.0.4 - hast-util-to-string@2.0.0: + hast-util-whitespace@3.0.0: dependencies: - '@types/hast': 2.3.10 - - hast-util-whitespace@2.0.1: {} + '@types/hast': 3.0.4 hastscript@7.2.0: dependencies: @@ -18721,19 +17794,20 @@ snapshots: property-information: 6.5.0 space-separated-tokens: 2.0.2 + hastscript@9.0.1: + dependencies: + '@types/hast': 3.0.4 + comma-separated-tokens: 2.0.3 + hast-util-parse-selector: 4.0.0 + property-information: 7.1.0 + space-separated-tokens: 2.0.2 + he@1.2.0: {} hexoid@2.0.0: {} highlight-words-core@1.2.3: {} - history@1.17.0: - dependencies: - deep-equal: 1.1.2 - invariant: 2.2.4 - query-string: 3.0.3 - warning: 2.1.0 - hoist-non-react-statics@3.3.2: dependencies: react-is: 16.13.1 @@ -18760,6 +17834,11 @@ snapshots: domhandler: 4.3.1 htmlparser2: 7.2.0 + html-element-map@1.3.1: + dependencies: + array.prototype.filter: 1.0.4 + call-bind: 1.0.8 + html-entities@2.6.0: {} html-escaper@2.0.2: {} @@ -18798,7 +17877,7 @@ snapshots: react-property: 2.0.0 style-to-js: 1.1.1 - html-void-elements@2.0.1: {} + html-void-elements@3.0.0: {} html-webpack-plugin@5.6.3(@rspack/core@1.3.15(@swc/helpers@0.5.5))(webpack@5.99.5): dependencies: @@ -18806,7 +17885,7 @@ snapshots: html-minifier-terser: 6.1.0 lodash: 4.17.21 pretty-error: 4.0.0 - tapable: 2.2.1 + tapable: 2.2.2 optionalDependencies: '@rspack/core': 1.3.15(@swc/helpers@0.5.5) webpack: 5.99.5 @@ -18832,8 +17911,6 @@ snapshots: domutils: 3.2.2 entities: 4.5.0 - htmlparser@1.7.7: {} - http-deceiver@1.2.7: {} http-errors@1.6.3: @@ -18855,7 +17932,7 @@ snapshots: http-proxy-3@1.20.5: dependencies: - debug: 4.4.1(supports-color@8.1.1) + debug: 4.4.1 follow-redirects: 1.15.9(debug@4.4.1) transitivePeerDependencies: - supports-color @@ -18864,14 +17941,14 @@ snapshots: dependencies: '@tootallnate/once': 2.0.0 agent-base: 6.0.2 - debug: 4.4.1(supports-color@8.1.1) + debug: 4.4.1 transitivePeerDependencies: - supports-color http-proxy-agent@7.0.2: dependencies: agent-base: 7.1.3 - debug: 4.4.1(supports-color@8.1.1) + debug: 4.4.1 transitivePeerDependencies: - supports-color @@ -18898,21 +17975,19 @@ snapshots: https-proxy-agent@5.0.1: dependencies: agent-base: 6.0.2 - debug: 4.4.1(supports-color@8.1.1) + debug: 4.4.1 transitivePeerDependencies: - supports-color https-proxy-agent@7.0.6: dependencies: agent-base: 7.1.3 - debug: 4.4.1(supports-color@8.1.1) + debug: 4.4.1 transitivePeerDependencies: - supports-color human-signals@2.1.0: {} - human-signals@5.0.0: {} - humanize-list@1.0.1: {} humanize-ms@1.2.1: @@ -19009,8 +18084,6 @@ snapshots: through: 2.3.8 wrap-ansi: 6.2.0 - install@0.13.0: {} - internal-slot@1.1.0: dependencies: es-errors: 1.3.0 @@ -19030,9 +18103,19 @@ snapshots: '@formatjs/icu-messageformat-parser': 2.11.2 tslib: 2.8.1 - invariant@2.2.4: - dependencies: - loose-envify: 1.4.0 + iovalkey@0.3.3: + dependencies: + '@iovalkey/commands': 0.1.0 + cluster-key-slot: 1.1.2 + debug: 4.4.1 + denque: 2.1.0 + lodash.defaults: 4.2.0 + lodash.isarguments: 3.1.0 + redis-errors: 1.2.0 + redis-parser: 3.0.0 + standard-as-callback: 2.1.0 + transitivePeerDependencies: + - supports-color ip-address@9.0.5: dependencies: @@ -19094,14 +18177,16 @@ snapshots: is-buffer@1.1.6: {} - is-buffer@2.0.5: {} - is-callable@1.2.7: {} is-core-module@2.15.1: dependencies: hasown: 2.0.2 + is-core-module@2.16.1: + dependencies: + hasown: 2.0.2 + is-data-view@1.0.2: dependencies: call-bound: 1.0.4 @@ -19227,8 +18312,6 @@ snapshots: is-stream@2.0.1: {} - is-stream@3.0.0: {} - is-string-blank@1.0.1: {} is-string@1.1.1: @@ -19236,6 +18319,8 @@ snapshots: call-bound: 1.0.4 has-tostringtag: 1.0.2 + is-subset@0.1.1: {} + is-svg-path@1.0.2: {} is-symbol@1.1.1: @@ -19288,7 +18373,7 @@ snapshots: istanbul-lib-instrument@5.2.1: dependencies: '@babel/core': 7.26.9 - '@babel/parser': 7.27.0 + '@babel/parser': 7.27.2 '@istanbuljs/schema': 0.1.3 istanbul-lib-coverage: 3.2.2 semver: 6.3.1 @@ -19298,7 +18383,7 @@ snapshots: istanbul-lib-instrument@6.0.3: dependencies: '@babel/core': 7.26.9 - '@babel/parser': 7.27.0 + '@babel/parser': 7.27.2 '@istanbuljs/schema': 0.1.3 istanbul-lib-coverage: 3.2.2 semver: 7.7.2 @@ -19313,7 +18398,7 @@ snapshots: istanbul-lib-source-maps@4.0.1: dependencies: - debug: 4.4.1(supports-color@8.1.1) + debug: 4.4.1 istanbul-lib-coverage: 3.2.2 source-map: 0.6.1 transitivePeerDependencies: @@ -19362,7 +18447,7 @@ snapshots: '@jest/expect': 29.7.0 '@jest/test-result': 29.7.0 '@jest/types': 29.6.3 - '@types/node': 24.0.1 + '@types/node': 18.19.111 chalk: 4.1.2 co: 4.6.0 dedent: 1.5.3 @@ -19511,7 +18596,7 @@ snapshots: '@jest/environment': 29.7.0 '@jest/fake-timers': 29.7.0 '@jest/types': 29.6.3 - '@types/node': 24.0.1 + '@types/node': 18.19.111 jest-mock: 29.7.0 jest-util: 29.7.0 @@ -19568,7 +18653,7 @@ snapshots: jest-message-util@29.7.0: dependencies: - '@babel/code-frame': 7.26.2 + '@babel/code-frame': 7.27.1 '@jest/types': 29.6.3 '@types/stack-utils': 2.0.3 chalk: 4.1.2 @@ -19581,7 +18666,7 @@ snapshots: jest-mock@29.7.0: dependencies: '@jest/types': 29.6.3 - '@types/node': 24.0.1 + '@types/node': 18.19.111 jest-util: 29.7.0 jest-pnp-resolver@1.2.3(jest-resolve@29.7.0): @@ -19607,7 +18692,7 @@ snapshots: jest-pnp-resolver: 1.2.3(jest-resolve@29.7.0) jest-util: 29.7.0 jest-validate: 29.7.0 - resolve: 1.22.8 + resolve: 1.22.10 resolve.exports: 2.0.2 slash: 3.0.0 @@ -19667,10 +18752,10 @@ snapshots: jest-snapshot@29.7.0: dependencies: '@babel/core': 7.26.9 - '@babel/generator': 7.27.0 + '@babel/generator': 7.27.1 '@babel/plugin-syntax-jsx': 7.27.1(@babel/core@7.26.9) - '@babel/plugin-syntax-typescript': 7.25.9(@babel/core@7.26.9) - '@babel/types': 7.27.0 + '@babel/plugin-syntax-typescript': 7.27.1(@babel/core@7.26.9) + '@babel/types': 7.27.1 '@jest/expect-utils': 29.7.0 '@jest/transform': 29.7.0 '@jest/types': 29.6.3 @@ -19720,13 +18805,13 @@ snapshots: jest-worker@27.5.1: dependencies: - '@types/node': 24.0.1 + '@types/node': 18.19.111 merge-stream: 2.0.0 supports-color: 8.1.1 jest-worker@29.7.0: dependencies: - '@types/node': 24.0.1 + '@types/node': 18.19.111 jest-util: 29.7.0 merge-stream: 2.0.0 supports-color: 8.1.1 @@ -19755,11 +18840,6 @@ snapshots: - supports-color - ts-node - jmp@2.0.0: - dependencies: - uuid: 3.4.0 - zeromq: 5.3.1 - jquery-focus-exit@1.0.1(jquery@3.7.1): dependencies: jquery: 3.7.1 @@ -19796,10 +18876,6 @@ snapshots: dependencies: jquery: 3.7.1 - jquery.payment@3.0.0: - dependencies: - jquery: 3.7.1 - jquery.scrollintoview@1.9.4: dependencies: jquery: 3.7.1 @@ -19900,7 +18976,7 @@ snapshots: lodash.isstring: 4.0.1 lodash.once: 4.1.1 ms: 2.1.2 - semver: 7.7.1 + semver: 7.7.2 jsx-ast-utils@3.3.5: dependencies: @@ -19917,8 +18993,6 @@ snapshots: dependencies: home-dir: 1.0.0 - just-extend@4.2.1: {} - jwa@1.4.1: dependencies: buffer-equal-constant-time: 1.0.1 @@ -19991,11 +19065,11 @@ snapshots: langs@2.0.0: {} - langsmith@0.3.31(openai@4.104.0(encoding@0.1.13)(ws@8.18.2)(zod@3.25.32)): + langsmith@0.3.29(openai@4.104.0(encoding@0.1.13)(ws@8.18.2)(zod@3.25.32)): dependencies: '@types/uuid': 10.0.0 chalk: 4.1.2 - console-table-printer: 2.14.3 + console-table-printer: 2.12.1 p-queue: 6.6.2 p-retry: 4.6.2 semver: 7.7.2 @@ -20016,13 +19090,6 @@ snapshots: dependencies: assert-plus: 1.0.0 - ldapauth-fork@5.0.5: - dependencies: - '@types/ldapjs': 2.2.5 - bcryptjs: 2.4.3 - ldapjs: 2.3.3 - lru-cache: 7.18.3 - ldapjs@2.3.3: dependencies: abstract-logging: 2.0.1 @@ -20032,16 +19099,7 @@ snapshots: ldap-filter: 0.3.3 once: 1.4.0 vasync: 2.2.1 - verror: 1.10.1 - - lean-client-js-core@1.5.0: - dependencies: - '@types/node': 9.6.61 - - lean-client-js-node@1.5.0: - dependencies: - '@types/node': 9.6.61 - lean-client-js-core: 1.5.0 + verror: 1.10.1 less-loader@11.1.4(less@4.3.0)(webpack@5.99.5): dependencies: @@ -20140,12 +19198,20 @@ snapshots: lodash.debounce@4.0.8: {} - lodash.get@4.4.2: {} + lodash.defaults@4.2.0: {} + + lodash.escape@4.0.1: {} + + lodash.flattendeep@4.4.0: {} lodash.includes@4.3.0: {} + lodash.isarguments@3.1.0: {} + lodash.isboolean@3.0.3: {} + lodash.isequal@4.5.0: {} + lodash.isinteger@4.0.4: {} lodash.isnumber@3.0.3: {} @@ -20171,12 +19237,6 @@ snapshots: chalk: 4.1.2 is-unicode-supported: 0.1.0 - lolex@2.7.5: {} - - lolex@5.1.2: - dependencies: - '@sinonjs/commons': 1.8.6 - long@5.3.1: {} loose-envify@1.4.0: @@ -20325,6 +19385,18 @@ snapshots: crypt: 0.0.2 is-buffer: 1.1.6 + mdast-util-to-hast@13.2.0: + dependencies: + '@types/hast': 3.0.4 + '@types/mdast': 4.0.4 + '@ungap/structured-clone': 1.3.0 + devlop: 1.1.0 + micromark-util-sanitize-uri: 2.0.1 + trim-lines: 3.0.1 + unist-util-position: 5.0.0 + unist-util-visit: 5.0.0 + vfile: 6.0.3 + mdurl@1.0.1: {} media-typer@0.3.0: {} @@ -20406,6 +19478,23 @@ snapshots: methods@1.1.2: {} + micromark-util-character@2.1.1: + dependencies: + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.2 + + micromark-util-encode@2.0.1: {} + + micromark-util-sanitize-uri@2.0.1: + dependencies: + micromark-util-character: 2.1.1 + micromark-util-encode: 2.0.1 + micromark-util-symbol: 2.0.1 + + micromark-util-symbol@2.0.1: {} + + micromark-util-types@2.0.2: {} + micromatch@4.0.8: dependencies: braces: 3.0.3 @@ -20425,8 +19514,6 @@ snapshots: mimic-fn@2.1.0: {} - mimic-fn@4.0.0: {} - mimic-response@3.1.0: {} min-document@2.19.0: @@ -20486,32 +19573,11 @@ snapshots: pkg-types: 1.3.1 ufo: 1.6.1 - mocha@10.8.2: - dependencies: - ansi-colors: 4.1.3 - browser-stdout: 1.3.1 - chokidar: 3.6.0 - debug: 4.4.1(supports-color@8.1.1) - diff: 5.2.0 - escape-string-regexp: 4.0.0 - find-up: 5.0.0 - glob: 8.1.0 - he: 1.2.0 - js-yaml: 4.1.0 - log-symbols: 4.1.0 - minimatch: 5.1.6 - ms: 2.1.3 - serialize-javascript: 6.0.2 - strip-json-comments: 3.1.1 - supports-color: 8.1.1 - workerpool: 6.5.1 - yargs: 16.2.0 - yargs-parser: 20.2.9 - yargs-unparser: 2.0.0 - moment@2.30.1: optional: true + moo@0.5.2: {} + mouse-change@1.4.0: dependencies: mouse-event: 1.0.5 @@ -20539,11 +19605,6 @@ snapshots: dns-packet: 5.6.1 thunky: 1.1.0 - multimath@2.0.0: - dependencies: - glur: 1.1.2 - object-assign: 4.1.1 - mumath@3.3.4: dependencies: almost-equal: 1.1.0 @@ -20561,29 +19622,28 @@ snapshots: rimraf: 2.4.5 optional: true - nan@2.17.0: {} - nan@2.20.0: {} nanoid@3.3.11: {} + nanoid@3.3.8: {} + napi-build-utils@2.0.0: {} native-promise-only@0.8.1: {} - nats.ws@1.30.3: - optionalDependencies: - nkeys.js: 1.1.0 - - nats@2.29.3: - dependencies: - nkeys.js: 1.1.0 - natural-compare@1.4.0: {} ncp@2.0.0: optional: true + nearley@2.20.1: + dependencies: + commander: 2.20.3 + moo: 0.5.2 + railroad-diagrams: 1.0.0 + randexp: 0.4.6 + needle@2.9.1: dependencies: debug: 3.2.7 @@ -20661,18 +19721,6 @@ snapshots: - '@babel/core' - babel-plugin-macros - nise@1.5.3: - dependencies: - '@sinonjs/formatio': 3.2.2 - '@sinonjs/text-encoding': 0.7.3 - just-extend: 4.2.1 - lolex: 5.1.2 - path-to-regexp: 1.9.0 - - nkeys.js@1.1.0: - dependencies: - tweetnacl: 1.0.3 - no-case@3.0.4: dependencies: lower-case: 2.0.2 @@ -20682,15 +19730,9 @@ snapshots: dependencies: semver: 7.7.2 - node-addon-api@6.1.0: {} - - node-addon-api@7.1.1: - optional: true + node-addon-api@7.1.1: {} - node-cjsx@2.0.0: - dependencies: - coffee-react-transform: 4.0.0 - coffee-script: 1.12.7 + node-addon-api@8.3.1: {} node-cleanup@2.1.2: {} @@ -20708,16 +19750,8 @@ snapshots: optionalDependencies: encoding: 0.1.13 - node-fetch@3.3.2: - dependencies: - data-uri-to-buffer: 4.0.1 - fetch-blob: 3.2.0 - formdata-polyfill: 4.0.10 - node-forge@1.3.1: {} - node-gyp-build@4.5.0: {} - node-int64@0.4.0: {} node-jose@2.2.0: @@ -20763,7 +19797,7 @@ snapshots: normalize-package-data@2.5.0: dependencies: hosted-git-info: 2.8.9 - resolve: 1.22.8 + resolve: 1.22.10 semver: 5.7.2 validate-npm-package-license: 3.0.4 @@ -20784,16 +19818,14 @@ snapshots: normalize-wheel@1.0.1: {} + notepack.io@3.0.1: {} + nouislider@15.4.0: {} npm-run-path@4.0.1: dependencies: path-key: 3.1.1 - npm-run-path@5.3.0: - dependencies: - path-key: 4.0.0 - nth-check@2.1.1: dependencies: boolbase: 1.0.0 @@ -20810,13 +19842,11 @@ snapshots: object-hash@3.0.0: {} - object-inspect@1.13.2: {} - object-inspect@1.13.4: {} object-is@1.1.6: dependencies: - call-bind: 1.0.7 + call-bind: 1.0.8 define-properties: 1.2.1 object-keys@1.1.1: {} @@ -20853,8 +19883,6 @@ snapshots: obuf@1.1.2: {} - octicons@3.5.0: {} - ollama@0.5.15: dependencies: whatwg-fetch: 3.6.20 @@ -20879,10 +19907,6 @@ snapshots: dependencies: mimic-fn: 2.1.0 - onetime@6.0.0: - dependencies: - mimic-fn: 4.0.0 - open@10.1.2: dependencies: default-browser: 5.2.1 @@ -20990,7 +20014,7 @@ snapshots: dependencies: '@tootallnate/quickjs-emscripten': 0.23.0 agent-base: 7.1.3 - debug: 4.4.1(supports-color@8.1.1) + debug: 4.4.1 get-uri: 6.0.4 http-proxy-agent: 7.0.2 https-proxy-agent: 7.0.6 @@ -21047,7 +20071,7 @@ snapshots: parse-json@5.2.0: dependencies: - '@babel/code-frame': 7.26.2 + '@babel/code-frame': 7.27.1 error-ex: 1.3.2 json-parse-even-better-errors: 2.3.1 lines-and-columns: 1.2.4 @@ -21073,10 +20097,14 @@ snapshots: parse5-htmlparser2-tree-adapter@7.1.0: dependencies: domhandler: 5.0.3 - parse5: 7.3.0 + parse5: 7.2.1 parse5@6.0.1: {} + parse5@7.2.1: + dependencies: + entities: 4.5.0 + parse5@7.3.0: dependencies: entities: 6.0.1 @@ -21130,11 +20158,6 @@ snapshots: dependencies: passport-oauth2: 1.8.0 - passport-ldapauth@3.0.1: - dependencies: - ldapauth-fork: 5.0.5 - passport-strategy: 1.0.0 - passport-oauth1@1.3.0: dependencies: oauth: 0.9.15 @@ -21193,8 +20216,6 @@ snapshots: path-key@3.1.1: {} - path-key@4.0.0: {} - path-parse@1.0.7: {} path-scurry@1.11.1: @@ -21204,10 +20225,6 @@ snapshots: path-to-regexp@0.1.12: {} - path-to-regexp@1.9.0: - dependencies: - isarray: 0.0.1 - path-to-regexp@8.2.0: {} path-type@4.0.0: {} @@ -21227,7 +20244,7 @@ snapshots: optionalDependencies: '@napi-rs/canvas': 0.1.69 - pegjs@0.10.0: {} + peek-readable@7.0.0: {} performance-now@2.1.0: {} @@ -21268,14 +20285,6 @@ snapshots: dependencies: split2: 4.2.0 - pica@7.1.1: - dependencies: - glur: 1.1.2 - inherits: 2.0.4 - multimath: 2.0.0 - object-assign: 4.1.1 - webworkify: 1.5.0 - pick-by-alias@1.2.0: {} picocolors@1.1.0: {} @@ -21286,8 +20295,6 @@ snapshots: picomatch@4.0.2: {} - pidusage@1.2.0: {} - pify@2.3.0: {} pify@4.0.1: {} @@ -21320,8 +20327,6 @@ snapshots: exsolve: 1.0.4 pathe: 2.0.3 - pkginfo@0.4.1: {} - playwright-core@1.51.1: optional: true @@ -21409,13 +20414,6 @@ snapshots: port-get@1.0.4: {} - portfinder@1.0.37: - dependencies: - async: 3.2.6 - debug: 4.4.1(supports-color@8.1.1) - transitivePeerDependencies: - - supports-color - possible-typed-array-names@1.1.0: {} postcss-modules-extract-imports@3.1.0(postcss@8.5.3): @@ -21532,10 +20530,12 @@ snapshots: eventemitter3: 5.0.1 forwarded-for: 1.1.0 fusing: 1.0.0 - nanoid: 3.3.11 + nanoid: 3.3.8 setheader: 1.0.2 ultron: 1.1.1 + printable-characters@1.0.42: {} + probe-image-size@7.2.3: dependencies: lodash.merge: 4.6.2 @@ -21570,11 +20570,9 @@ snapshots: property-information@6.5.0: {} - proto3-json-serializer@2.0.2: - dependencies: - protobufjs: 7.4.0 + property-information@7.1.0: {} - proto3-json-serializer@3.0.0: + proto3-json-serializer@2.0.2: dependencies: protobufjs: 7.4.0 @@ -21603,7 +20601,7 @@ snapshots: proxy-agent@6.5.0: dependencies: agent-base: 7.1.3 - debug: 4.4.1(supports-color@8.1.1) + debug: 4.4.1 http-proxy-agent: 7.0.2 https-proxy-agent: 7.0.6 lru-cache: 7.18.3 @@ -21633,14 +20631,10 @@ snapshots: qs@6.13.0: dependencies: - side-channel: 1.0.6 + side-channel: 1.1.0 quansync@0.2.10: {} - query-string@3.0.3: - dependencies: - strict-uri-encode: 1.1.0 - querystringify@2.2.0: {} queue-microtask@1.2.3: {} @@ -21659,6 +20653,13 @@ snapshots: dependencies: performance-now: 2.1.0 + railroad-diagrams@1.0.0: {} + + randexp@0.4.6: + dependencies: + discontinuous-range: 1.0.0 + ret: 0.1.15 + random-bytes@1.0.0: {} random-key@0.3.2: {} @@ -21841,7 +20842,7 @@ snapshots: rc-overflow@1.4.1(react-dom@18.3.1(react@18.3.1))(react@18.3.1): dependencies: - '@babel/runtime': 7.27.0 + '@babel/runtime': 7.27.1 classnames: 2.5.1 rc-resize-observer: 1.4.3(react-dom@18.3.1(react@18.3.1))(react@18.3.1) rc-util: 5.44.4(react-dom@18.3.1(react@18.3.1))(react@18.3.1) @@ -22028,7 +21029,7 @@ snapshots: rc-virtual-list@3.18.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1): dependencies: - '@babel/runtime': 7.27.0 + '@babel/runtime': 7.27.1 classnames: 2.5.1 rc-resize-observer: 1.4.3(react-dom@18.3.1(react@18.3.1))(react@18.3.1) rc-util: 5.44.4(react-dom@18.3.1(react@18.3.1))(react@18.3.1) @@ -22037,7 +21038,7 @@ snapshots: rc-virtual-list@3.18.6(react-dom@18.3.1(react@18.3.1))(react@18.3.1): dependencies: - '@babel/runtime': 7.27.0 + '@babel/runtime': 7.27.1 classnames: 2.5.1 rc-resize-observer: 1.4.3(react-dom@18.3.1(react@18.3.1))(react@18.3.1) rc-util: 5.44.4(react-dom@18.3.1(react@18.3.1))(react@18.3.1) @@ -22056,15 +21057,9 @@ snapshots: react: 18.3.1 react-dom: 18.3.1(react@18.3.1) - react-async-script@1.2.0(react@18.3.1): - dependencies: - hoist-non-react-statics: 3.3.2 - prop-types: 15.8.1 - react: 18.3.1 - react-base16-styling@0.9.1: dependencies: - '@babel/runtime': 7.27.0 + '@babel/runtime': 7.27.1 '@types/base16': 1.0.5 '@types/lodash': 4.17.17 base16: 1.0.0 @@ -22130,12 +21125,6 @@ snapshots: react: 18.3.1 react-dom: 18.3.1(react@18.3.1) - react-google-recaptcha@2.1.0(react@18.3.1): - dependencies: - prop-types: 15.8.1 - react: 18.3.1 - react-async-script: 1.2.0(react@18.3.1) - react-helmet@6.1.0(react@18.3.1): dependencies: object-assign: 4.1.1 @@ -22161,7 +21150,7 @@ snapshots: '@formatjs/icu-messageformat-parser': 2.11.2 '@formatjs/intl': 3.1.6(typescript@5.8.3) '@types/hoist-non-react-statics': 3.3.1 - '@types/react': 18.3.10 + '@types/react': 18.3.23 hoist-non-react-statics: 3.3.2 intl-messageformat: 10.7.16 react: 18.3.1 @@ -22221,7 +21210,7 @@ snapshots: react-textarea-autosize@8.5.9(@types/react@18.3.23)(react@18.3.1): dependencies: - '@babel/runtime': 7.27.0 + '@babel/runtime': 7.27.1 react: 18.3.1 use-composed-ref: 1.4.0(@types/react@18.3.23)(react@18.3.1) use-latest: 1.3.0(@types/react@18.3.23)(react@18.3.1) @@ -22294,13 +21283,19 @@ snapshots: rechoir@0.8.0: dependencies: - resolve: 1.22.8 + resolve: 1.22.10 redent@3.0.0: dependencies: indent-string: 4.0.0 strip-indent: 3.0.0 + redis-errors@1.2.0: {} + + redis-parser@3.0.0: + dependencies: + redis-errors: 1.2.0 + redux@4.2.1: dependencies: '@babel/runtime': 7.25.6 @@ -22393,34 +21388,33 @@ snapshots: regl@2.1.1: {} - rehype-parse@8.0.5: + rehype-parse@9.0.1: dependencies: - '@types/hast': 2.3.10 - hast-util-from-parse5: 7.1.2 - parse5: 6.0.1 - unified: 10.1.2 + '@types/hast': 3.0.4 + hast-util-from-html: 2.0.3 + unified: 11.0.5 - rehype-prism-plus@1.6.3: + rehype-prism-plus@2.0.0: dependencies: - hast-util-to-string: 2.0.0 + hast-util-to-string: 3.0.1 parse-numeric-range: 1.3.0 refractor: 4.9.0 - rehype-parse: 8.0.5 - unist-util-filter: 4.0.1 - unist-util-visit: 4.1.2 + rehype-parse: 9.0.1 + unist-util-filter: 5.0.1 + unist-util-visit: 5.0.0 - rehype-stringify@9.0.4: + rehype-stringify@10.0.1: dependencies: - '@types/hast': 2.3.10 - hast-util-to-html: 8.0.4 - unified: 10.1.2 + '@types/hast': 3.0.4 + hast-util-to-html: 9.0.5 + unified: 11.0.5 - rehype@12.0.1: + rehype@13.0.2: dependencies: - '@types/hast': 2.3.10 - rehype-parse: 8.0.5 - rehype-stringify: 9.0.4 - unified: 10.1.2 + '@types/hast': 3.0.4 + rehype-parse: 9.0.1 + rehype-stringify: 10.0.1 + unified: 11.0.5 relateurl@0.2.7: {} @@ -22438,8 +21432,6 @@ snapshots: require-from-string@2.0.2: {} - require-reload@0.2.2: {} - requires-port@1.0.0: {} reselect@4.1.8: {} @@ -22462,6 +21454,12 @@ snapshots: resolve@0.6.3: {} + resolve@1.22.10: + dependencies: + is-core-module: 2.16.1 + path-parse: 1.0.7 + supports-preserve-symlinks-flag: 1.0.0 + resolve@1.22.8: dependencies: is-core-module: 2.15.1 @@ -22479,6 +21477,8 @@ snapshots: onetime: 5.1.2 signal-exit: 3.0.7 + ret@0.1.15: {} + retry-request@7.0.2(encoding@0.1.13): dependencies: '@types/request': 2.48.12 @@ -22488,14 +21488,6 @@ snapshots: - encoding - supports-color - retry-request@8.0.0: - dependencies: - '@types/request': 2.48.12 - extend: 3.0.2 - teeny-request: 10.1.0 - transitivePeerDependencies: - - supports-color - retry@0.13.1: {} reusify@1.0.4: {} @@ -22528,6 +21520,11 @@ snapshots: points-on-curve: 0.2.0 points-on-path: 0.2.1 + rst-selector-parser@2.2.3: + dependencies: + lodash.flattendeep: 4.4.0 + nearley: 2.20.1 + run-applescript@7.0.0: {} run-async@2.4.1: {} @@ -22573,8 +21570,6 @@ snapshots: safer-buffer@2.1.2: {} - samsam@1.3.0: {} - sanitize-html@2.17.0: dependencies: deepmerge: 4.3.1 @@ -22740,19 +21735,6 @@ snapshots: shallowequal@1.1.0: {} - sharp@0.32.6: - dependencies: - color: 4.2.3 - detect-libc: 2.0.3 - node-addon-api: 6.1.0 - prebuild-install: 7.1.3 - semver: 7.7.1 - simple-get: 4.0.1 - tar-fs: 3.0.9 - tunnel-agent: 0.6.0 - transitivePeerDependencies: - - bare-buffer - shebang-command@2.0.0: dependencies: shebang-regex: 3.0.0 @@ -22763,28 +21745,8 @@ snapshots: shell-quote@1.8.3: {} - should-equal@0.5.0: - dependencies: - should-type: 0.2.0 - - should-format@0.3.1: - dependencies: - should-type: 0.2.0 - should-proxy@1.0.4: {} - should-sinon@0.0.3(should@7.1.1): - dependencies: - should: 7.1.1 - - should-type@0.2.0: {} - - should@7.1.1: - dependencies: - should-equal: 0.5.0 - should-format: 0.3.1 - should-type: 0.2.0 - side-channel-list@1.0.0: dependencies: es-errors: 1.3.0 @@ -22794,14 +21756,14 @@ snapshots: dependencies: call-bound: 1.0.4 es-errors: 1.3.0 - get-intrinsic: 1.2.7 + get-intrinsic: 1.3.0 object-inspect: 1.13.4 side-channel-weakmap@1.0.2: dependencies: call-bound: 1.0.4 es-errors: 1.3.0 - get-intrinsic: 1.2.7 + get-intrinsic: 1.3.0 object-inspect: 1.13.4 side-channel-map: 1.0.1 @@ -22810,7 +21772,7 @@ snapshots: call-bind: 1.0.8 es-errors: 1.3.0 get-intrinsic: 1.3.0 - object-inspect: 1.13.2 + object-inspect: 1.13.4 side-channel@1.1.0: dependencies: @@ -22840,16 +21802,6 @@ snapshots: simple-wcswidth@1.0.1: {} - sinon@4.5.0: - dependencies: - '@sinonjs/formatio': 2.0.0 - diff: 3.5.0 - lodash.get: 4.4.2 - lolex: 2.7.5 - nise: 1.5.3 - supports-color: 5.5.0 - type-detect: 4.1.0 - sirv@2.0.4: dependencies: '@polka/url': 1.0.0-next.29 @@ -22876,6 +21828,47 @@ snapshots: smart-buffer@4.2.0: {} + socket.io-adapter@2.5.5: + dependencies: + debug: 4.3.7 + ws: 8.17.1 + transitivePeerDependencies: + - bufferutil + - supports-color + - utf-8-validate + + socket.io-client@4.8.1: + dependencies: + '@socket.io/component-emitter': 3.1.2 + debug: 4.3.7 + engine.io-client: 6.6.3 + socket.io-parser: 4.2.4 + transitivePeerDependencies: + - bufferutil + - supports-color + - utf-8-validate + + socket.io-parser@4.2.4: + dependencies: + '@socket.io/component-emitter': 3.1.2 + debug: 4.3.7 + transitivePeerDependencies: + - supports-color + + socket.io@4.8.1: + dependencies: + accepts: 1.3.8 + base64id: 2.0.0 + cors: 2.8.5 + debug: 4.3.7 + engine.io: 6.6.4 + socket.io-adapter: 2.5.5 + socket.io-parser: 4.2.4 + transitivePeerDependencies: + - bufferutil + - supports-color + - utf-8-validate + sockjs@0.3.24: dependencies: faye-websocket: 0.11.4 @@ -22885,7 +21878,7 @@ snapshots: socks-proxy-agent@8.0.5: dependencies: agent-base: 7.1.3 - debug: 4.4.1(supports-color@8.1.1) + debug: 4.4.1 socks: 2.8.4 transitivePeerDependencies: - supports-color @@ -22936,7 +21929,7 @@ snapshots: spdy-transport@3.0.0: dependencies: - debug: 4.4.1(supports-color@8.1.1) + debug: 4.4.1 detect-node: 2.1.0 hpack.js: 2.1.6 obuf: 1.1.2 @@ -22947,7 +21940,7 @@ snapshots: spdy@4.0.2: dependencies: - debug: 4.4.1(supports-color@8.1.1) + debug: 4.4.1 handle-thing: 2.0.1 http-deceiver: 1.2.7 select-hose: 2.0.0 @@ -22971,6 +21964,8 @@ snapshots: stackframe@1.3.4: {} + standard-as-callback@2.1.0: {} + static-eval@2.1.1: dependencies: escodegen: 2.1.0 @@ -23003,15 +21998,6 @@ snapshots: streamsearch@1.1.0: {} - streamx@2.22.0: - dependencies: - fast-fifo: 1.3.2 - text-decoder: 1.2.3 - optionalDependencies: - bare-events: 2.5.4 - - strict-uri-encode@1.1.0: {} - string-convert@0.2.1: {} string-length@4.0.2: @@ -23108,8 +22094,6 @@ snapshots: strip-final-newline@2.0.0: {} - strip-final-newline@3.0.0: {} - strip-indent@3.0.0: dependencies: min-indent: 1.0.1 @@ -23127,9 +22111,10 @@ snapshots: strongly-connected-components@1.0.1: {} - strtok3@10.3.1: + strtok3@10.2.2: dependencies: '@tokenizer/token': 0.3.0 + peek-readable: 7.0.0 stubs@3.0.0: {} @@ -23186,8 +22171,6 @@ snapshots: dependencies: has-flag: 4.0.0 - supports-color@9.4.0: {} - supports-hyperlinks@2.2.0: dependencies: has-flag: 4.0.0 @@ -23224,8 +22207,6 @@ snapshots: string-width: 4.2.3 strip-ansi: 6.0.1 - tapable@2.2.1: {} - tapable@2.2.2: {} tar-fs@2.1.3: @@ -23235,16 +22216,6 @@ snapshots: pump: 3.0.2 tar-stream: 2.2.0 - tar-fs@3.0.9: - dependencies: - pump: 3.0.2 - tar-stream: 3.1.7 - optionalDependencies: - bare-fs: 4.1.2 - bare-path: 3.0.0 - transitivePeerDependencies: - - bare-buffer - tar-stream@2.2.0: dependencies: bl: 4.1.0 @@ -23253,25 +22224,10 @@ snapshots: inherits: 2.0.4 readable-stream: 3.6.2 - tar-stream@3.1.7: - dependencies: - b4a: 1.6.7 - fast-fifo: 1.3.2 - streamx: 2.22.0 - tdigest@0.1.2: dependencies: bintrees: 1.0.2 - teeny-request@10.1.0: - dependencies: - http-proxy-agent: 5.0.0 - https-proxy-agent: 5.0.1 - node-fetch: 3.3.2 - stream-events: 1.0.5 - transitivePeerDependencies: - - supports-color - teeny-request@9.0.0(encoding@0.1.13): dependencies: http-proxy-agent: 5.0.0 @@ -23335,10 +22291,6 @@ snapshots: glob: 7.2.3 minimatch: 3.1.2 - text-decoder@1.2.3: - dependencies: - b4a: 1.6.7 - text-hex@1.0.0: {} text-table@0.2.0: {} @@ -23384,9 +22336,9 @@ snapshots: tinyexec@0.3.2: {} - tinyglobby@0.2.14: + tinyglobby@0.2.13: dependencies: - fdir: 6.4.6(picomatch@4.0.2) + fdir: 6.4.4(picomatch@4.0.2) picomatch: 4.0.2 tinyqueue@2.0.3: {} @@ -23438,6 +22390,8 @@ snapshots: tree-kill@1.2.2: {} + trim-lines@3.0.1: {} + trim-newlines@3.0.1: {} trough@2.2.0: {} @@ -23497,8 +22451,6 @@ snapshots: path-exists: 4.0.0 read-pkg-up: 7.0.1 - tsimportlib@0.0.5: {} - tslib@1.14.1: {} tslib@2.8.1: {} @@ -23509,16 +22461,12 @@ snapshots: dependencies: safe-buffer: 5.2.1 - tweetnacl@1.0.3: {} - type-check@0.4.0: dependencies: prelude-ls: 1.2.1 type-detect@4.0.8: {} - type-detect@4.1.0: {} - type-fest@0.13.1: {} type-fest@0.18.1: {} @@ -23598,6 +22546,8 @@ snapshots: uid2@0.0.4: {} + uid2@1.0.0: {} + uid@2.0.2: dependencies: '@lukeed/csprng': 1.1.0 @@ -23621,15 +22571,15 @@ snapshots: unicorn-magic@0.1.0: {} - unified@10.1.2: + unified@11.0.5: dependencies: - '@types/unist': 2.0.11 + '@types/unist': 3.0.3 bail: 2.0.2 + devlop: 1.1.0 extend: 3.0.2 - is-buffer: 2.0.5 is-plain-obj: 4.1.0 trough: 2.2.0 - vfile: 5.3.7 + vfile: 6.0.3 unique-random-array@1.0.1: dependencies: @@ -23643,34 +22593,34 @@ snapshots: unique-random@2.1.0: {} - unist-util-filter@4.0.1: + unist-util-filter@5.0.1: dependencies: - '@types/unist': 2.0.11 - unist-util-is: 5.2.1 - unist-util-visit-parents: 5.1.3 + '@types/unist': 3.0.3 + unist-util-is: 6.0.0 + unist-util-visit-parents: 6.0.1 - unist-util-is@5.2.1: + unist-util-is@6.0.0: dependencies: - '@types/unist': 2.0.11 + '@types/unist': 3.0.3 - unist-util-position@4.0.4: + unist-util-position@5.0.0: dependencies: - '@types/unist': 2.0.11 + '@types/unist': 3.0.3 - unist-util-stringify-position@3.0.3: + unist-util-stringify-position@4.0.0: dependencies: - '@types/unist': 2.0.11 + '@types/unist': 3.0.3 - unist-util-visit-parents@5.1.3: + unist-util-visit-parents@6.0.1: dependencies: - '@types/unist': 2.0.11 - unist-util-is: 5.2.1 + '@types/unist': 3.0.3 + unist-util-is: 6.0.0 - unist-util-visit@4.1.2: + unist-util-visit@5.0.0: dependencies: - '@types/unist': 2.0.11 - unist-util-is: 5.2.1 - unist-util-visit-parents: 5.1.3 + '@types/unist': 3.0.3 + unist-util-is: 6.0.0 + unist-util-visit-parents: 6.0.1 universal-cookie@4.0.4: dependencies: @@ -23685,9 +22635,9 @@ snapshots: unzip-response@2.0.1: {} - update-browserslist-db@1.1.3(browserslist@4.24.4): + update-browserslist-db@1.1.3(browserslist@4.24.5): dependencies: - browserslist: 4.24.4 + browserslist: 4.24.5 escalade: 3.2.0 picocolors: 1.1.1 @@ -23786,10 +22736,6 @@ snapshots: uuid@11.1.0: {} - uuid@3.4.0: {} - - uuid@7.0.3: {} - uuid@8.3.2: {} uuid@9.0.1: {} @@ -23842,22 +22788,20 @@ snapshots: core-util-is: 1.0.2 extsprintf: 1.4.1 - vfile-location@4.1.0: + vfile-location@5.0.3: dependencies: - '@types/unist': 2.0.11 - vfile: 5.3.7 + '@types/unist': 3.0.3 + vfile: 6.0.3 - vfile-message@3.1.4: + vfile-message@4.0.2: dependencies: - '@types/unist': 2.0.11 - unist-util-stringify-position: 3.0.3 + '@types/unist': 3.0.3 + unist-util-stringify-position: 4.0.0 - vfile@5.3.7: + vfile@6.0.3: dependencies: - '@types/unist': 2.0.11 - is-buffer: 2.0.5 - unist-util-stringify-position: 3.0.3 - vfile-message: 3.1.4 + '@types/unist': 3.0.3 + vfile-message: 4.0.2 video-extensions@1.2.0: {} @@ -23894,10 +22838,6 @@ snapshots: dependencies: makeerror: 1.0.12 - warning@2.1.0: - dependencies: - loose-envify: 1.4.0 - warning@4.0.3: dependencies: loose-envify: 1.4.0 @@ -23919,8 +22859,6 @@ snapshots: web-namespaces@2.0.1: {} - web-streams-polyfill@3.3.3: {} - web-streams-polyfill@4.0.0-beta.3: {} webgl-context@2.2.0: @@ -24090,7 +23028,7 @@ snapshots: websocket-sftp@0.8.4: dependencies: awaiting: 3.0.0 - debug: 4.4.1(supports-color@8.1.1) + debug: 4.4.1 port-get: 1.0.4 ws: 8.18.2 transitivePeerDependencies: @@ -24098,8 +23036,6 @@ snapshots: - supports-color - utf-8-validate - webworkify@1.5.0: {} - whatwg-fetch@3.6.20: {} whatwg-url@5.0.0: @@ -24160,8 +23096,6 @@ snapshots: wordwrap@1.0.0: {} - workerpool@6.5.1: {} - world-calendars@1.0.3: dependencies: object-assign: 4.1.1 @@ -24193,6 +23127,8 @@ snapshots: ws@7.5.10: {} + ws@8.17.1: {} + ws@8.18.2: {} xdg-basedir@5.1.0: {} @@ -24225,6 +23161,8 @@ snapshots: xmlbuilder@15.1.1: {} + xmlhttprequest-ssl@2.1.2: {} + xpath@0.0.32: {} xpath@0.0.33: {} @@ -24240,20 +23178,6 @@ snapshots: xtend@4.0.2: {} - xterm-addon-fit@0.6.0(xterm@5.0.0): - dependencies: - xterm: 5.0.0 - - xterm-addon-web-links@0.7.0(xterm@5.0.0): - dependencies: - xterm: 5.0.0 - - xterm-addon-webgl@0.13.0(xterm@5.0.0): - dependencies: - xterm: 5.0.0 - - xterm@5.0.0: {} - y-protocols@1.0.6(yjs@13.6.24): dependencies: lib0: 0.2.102 @@ -24276,13 +23200,6 @@ snapshots: yargs-parser@21.1.1: {} - yargs-unparser@2.0.0: - dependencies: - camelcase: 6.3.0 - decamelize: 4.0.0 - flat: 5.0.2 - is-plain-obj: 2.1.0 - yargs@16.2.0: dependencies: cliui: 7.0.4 @@ -24311,10 +23228,10 @@ snapshots: yocto-queue@1.2.1: {} - zeromq@5.3.1: + zeromq@6.4.2: dependencies: - nan: 2.17.0 - node-gyp-build: 4.5.0 + cmake-ts: 0.6.1 + node-addon-api: 8.3.1 zlibjs@0.3.1: {} @@ -24334,4 +23251,10 @@ snapshots: zod@3.25.64: {} + zstd-napi@0.0.10: + dependencies: + '@types/node': 18.19.111 + node-addon-api: 7.1.1 + prebuild-install: 7.1.3 + zwitch@2.0.4: {} diff --git a/src/packages/pnpm-workspace.yaml b/src/packages/pnpm-workspace.yaml index 0e2c3f64b8..06d7a26852 100644 --- a/src/packages/pnpm-workspace.yaml +++ b/src/packages/pnpm-workspace.yaml @@ -5,3 +5,8 @@ packages: # weird build system, and it totally messes up stuff. - "!**/cdn/dist/**" - "!compute" +onlyBuiltDependencies: + - better-sqlite3 + - websocket-sftp + - websocketfs + - zstd-napi diff --git a/src/packages/project/bin/run-project.py b/src/packages/project/bin/run-project.py index 86343d94ff..75179d5a97 100755 --- a/src/packages/project/bin/run-project.py +++ b/src/packages/project/bin/run-project.py @@ -14,13 +14,8 @@ ./run-project.py /path/to/launch-params.json -DO NOT expect to interact with a project running this way from your -frontend browser client. That's not going to work, because the hub -will try to start the project again (thus deleting and recreating ~/.smc). The purpose of this script is just to help in figuring out why a project starts up and then JUST CRASHES for mysterious reasons. - - **It won't help with anything else yet.** """ import json, subprocess, os, sys @@ -37,17 +32,21 @@ def run_command_with_params(params): # Get the command, args, and cwd from the params cmd = params["cmd"] args = params["args"] - args = [x for x in args if 'daemon' not in x] + #args = [x for x in args if 'daemon' not in x] cwd = params["cwd"] # Get the environment variables from the params env = params["env"] - env['DEBUG'] = '*' + if 'DEBUG' not in env: + env['DEBUG'] = 'cocalc:*' env['DEBUG_CONSOLE'] = 'yes' # Convert the environment dictionary to a list of key=value strings env_list = [f"{key}={value}" for key, value in env.items()] + print( + "Running the following command with the environment setup for the project:\n" + ) print(" ".join([cmd] + args)) try: # Run the command with the specified arguments and environment in the given cwd @@ -71,6 +70,6 @@ def run_command_with_params(params): params = json.load(file) run_command_with_params(params) except FileNotFoundError: - print("File 'sys.argv[1]' not found.") + print(f"File '{sys.argv[1]}' not found.") except json.JSONDecodeError: print("Error parsing JSON data from the file.") diff --git a/src/packages/project/browser-websocket/api.ts b/src/packages/project/browser-websocket/api.ts index 67f2149e3d..8ca6e9387b 100644 --- a/src/packages/project/browser-websocket/api.ts +++ b/src/packages/project/browser-websocket/api.ts @@ -18,12 +18,8 @@ import { getClient } from "@cocalc/project/client"; import { get_configuration } from "../configuration"; import { run_formatter, run_formatter_string } from "../formatters"; import { nbconvert as jupyter_nbconvert } from "../jupyter/convert"; -import { lean, lean_channel } from "../lean/server"; import { jupyter_strip_notebook } from "@cocalc/jupyter/nbgrader/jupyter-parse"; import { jupyter_run_notebook } from "@cocalc/jupyter/nbgrader/jupyter-run"; -import { synctable_channel } from "../sync/server"; -import { syncdoc_call } from "../sync/sync-doc"; -import { terminal } from "@cocalc/terminal"; import { x11_channel } from "../x11/server"; import { canonical_paths } from "./canonical-path"; import { delete_files } from "@cocalc/backend/files/delete-files"; @@ -32,9 +28,7 @@ import computeFilesystemCache from "./compute-filesystem-cache"; import { move_files } from "@cocalc/backend/files/move-files"; import { rename_file } from "@cocalc/backend/files/rename-file"; import { realpath } from "./realpath"; -import { project_info_ws } from "../project-info"; import query from "./query"; -import { browser_symmetric_channel } from "./symmetric_channel"; import type { Mesg } from "@cocalc/comm/websocket/types"; import handleSyncFsApiCall, { handleSyncFsRequestCall, @@ -158,9 +152,6 @@ export async function handleApiCall({ case "eval_code": return await eval_code(data.code); - case "terminal": - return await terminal(primus, data.path, data.options); - case "jupyter_strip_notebook": return await jupyter_strip_notebook(data.ipynb_path); case "jupyter_nbconvert": @@ -168,30 +159,9 @@ export async function handleApiCall({ case "jupyter_run_notebook": return await jupyter_run_notebook(data.opts); - case "lean": - return await lean(client, primus, log, data.opts); - case "lean_channel": - return await lean_channel(client, primus, log, data.path); - case "x11_channel": return await x11_channel(client, primus, log, data.path, data.display); - case "synctable_channel": - return await synctable_channel( - client, - primus, - log, - data.query, - data.options, - ); - case "syncdoc_call": - return await syncdoc_call(data.path, data.mesg); - case "symmetric_channel": - return await browser_symmetric_channel(client, primus, log, data.name); - - case "project_info": - return await project_info_ws(primus, log); - // compute server case "compute_filesystem_cache": diff --git a/src/packages/project/browser-websocket/server.ts b/src/packages/project/browser-websocket/server.ts index 93c6f3a79c..74deb5b2d6 100644 --- a/src/packages/project/browser-websocket/server.ts +++ b/src/packages/project/browser-websocket/server.ts @@ -11,8 +11,7 @@ import { join } from "node:path"; import { Router } from "express"; import { Server } from "http"; import Primus from "primus"; -import type { PrimusWithChannels } from "@cocalc/terminal"; -import initNats from "@cocalc/project/nats"; +import initConat from "@cocalc/project/conat"; // We are NOT using UglifyJS because it can easily take 3 blocking seconds of cpu // during project startup to save 100kb -- it just isn't worth it. Obviously, it @@ -29,7 +28,7 @@ export default function init(server: Server, basePath: string): Router { transformer: "websockets", } as const; winston.info(`Initializing primus websocket server at "${opts.pathname}"...`); - const primus = new Primus(server, opts) as PrimusWithChannels; + const primus = new Primus(server, opts); // add multiplex to Primus so we have channels. primus.plugin("multiplex", require("@cocalc/primus-multiplex")); @@ -57,8 +56,8 @@ export default function init(server: Server, basePath: string): Router { `waiting for clients to request primus.js (length=${library.length})...`, ); - // we also init the new nats server, which is meant to replace this: - initNats(); + // we also init the conat server, which is meant to replace this: + initConat(); return router; } diff --git a/src/packages/project/browser-websocket/symmetric_channel.ts b/src/packages/project/browser-websocket/symmetric_channel.ts deleted file mode 100644 index 9a2cd9594c..0000000000 --- a/src/packages/project/browser-websocket/symmetric_channel.ts +++ /dev/null @@ -1,103 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -/* -Channels used for maybe nothing right now. - -I thought this would be useful, but it hasn't yet turned out to be. -*/ - -import { EventEmitter } from "events"; - -const sync_tables = {}; - -function get_name(name: string): string { - return `symmetric_channel:${name}`; -} - -export async function browser_symmetric_channel( - _: any, - primus: any, - logger: any, - name: string -): Promise { - name = get_name(name); - - // The code below is way more complicated because SymmetricChannel - // can be made *before* this sync function is called. If that - // happens, and we also have to set the channel of SymmetricChannel. - - if ( - sync_tables[name] !== undefined && - sync_tables[name].channel !== undefined - ) { - // fully initialized - return name; - } - - const channel = primus.channel(name); - let local: SymmetricChannel; - - if (sync_tables[name] !== undefined) { - local = sync_tables[name].local; - local.channel = channel; - sync_tables[name].channel = channel; - } else { - local = new SymmetricChannel(channel); - sync_tables[name] = { - local, - channel, - }; - } - - channel.on("connection", function (spark: any): void { - // Now handle a connection - logger.debug("sync", name, `conn from ${spark.address.ip} -- ${spark.id}`); - spark.on("end", function () { - logger.debug("sync", name, `closed ${spark.address.ip} -- ${spark.id}`); - }); - spark.on("data", function (data) { - local._data_from_spark(data); - channel.forEach(function (spark0, id) { - if (id !== spark.id) { - spark0.write(data); - } - }); - }); - }); - - return name; -} - -class SymmetricChannel extends EventEmitter { - channel: any; - - constructor(channel?: any) { - super(); - this.channel = channel; - } - - // Returns true if immediate write succeeds - write(data: any): boolean { - if (this.channel !== undefined) { - return this.channel.write(data); - } - return false; - } - - _data_from_spark(data: any): void { - this.emit("data", data); - } -} - -export function symmetric_channel(name: string): SymmetricChannel { - name = get_name(name); - if (sync_tables[name] !== undefined) { - return sync_tables[name].local; - } - const local = new SymmetricChannel(undefined); - sync_tables[name] = { local }; - return local; -} diff --git a/src/packages/project/bug-counter.ts b/src/packages/project/bug-counter.ts index 8a069bade7..41c50adc42 100644 --- a/src/packages/project/bug-counter.ts +++ b/src/packages/project/bug-counter.ts @@ -25,6 +25,7 @@ export function init() { const border = `BUG (count=${bugCount}) ${STARS}`; log.error(border); log.error(`Uncaught exception: ${err}`); + console.warn(err); log.error(err.stack); log.error(border); }; diff --git a/src/packages/project/client.ts b/src/packages/project/client.ts index f8a19fdf72..ea631ddc47 100644 --- a/src/packages/project/client.ts +++ b/src/packages/project/client.ts @@ -31,34 +31,31 @@ import type { SyncDoc } from "@cocalc/sync/editor/generic/sync-doc"; import type { ProjectClient as ProjectClientInterface } from "@cocalc/sync/editor/generic/types"; import { SyncString } from "@cocalc/sync/editor/string/sync"; import * as synctable2 from "@cocalc/sync/table"; -import { callback2, once } from "@cocalc/util/async-utils"; +import { callback2 } from "@cocalc/util/async-utils"; import { PROJECT_HUB_HEARTBEAT_INTERVAL_S } from "@cocalc/util/heartbeat"; import * as message from "@cocalc/util/message"; import * as misc from "@cocalc/util/misc"; import type { CB } from "@cocalc/util/types/callback"; import type { ExecuteCodeOptionsWithCallback } from "@cocalc/util/types/execute-code"; import * as blobs from "./blobs"; -import { symmetric_channel } from "./browser-websocket/symmetric_channel"; import { json } from "./common"; import * as data from "./data"; import initJupyter from "./jupyter/init"; import * as kucalc from "./kucalc"; import { getLogger } from "./logger"; import * as sage_session from "./sage_session"; -import { getListingsTable } from "@cocalc/project/sync/listings"; -import { get_synctable } from "./sync/open-synctables"; -import { get_syncdoc } from "./sync/sync-doc"; -import synctable_nats from "@cocalc/project/nats/synctable"; -import pubsub from "@cocalc/project/nats/pubsub"; -import type { NatsSyncTableFunction } from "@cocalc/nats/sync/synctable"; -import { getEnv as getNatsEnv } from "@cocalc/project/nats/env"; +import synctable_conat from "@cocalc/project/conat/synctable"; +import pubsub from "@cocalc/project/conat/pubsub"; +import type { ConatSyncTableFunction } from "@cocalc/conat/sync/synctable"; import { - callNatsService, - createNatsService, - type CallNatsServiceFunction, - type CreateNatsServiceFunction, -} from "@cocalc/nats/service"; -import type { NatsEnvFunction } from "@cocalc/nats/types"; + callConatService, + createConatService, + type CallConatServiceFunction, + type CreateConatServiceFunction, +} from "@cocalc/conat/service"; +import { connectToConat } from "./conat/connection"; +import { getSyncDoc } from "@cocalc/project/conat/open-files"; +import { isDeleted } from "@cocalc/project/conat/listings"; const winston = getLogger("client"); @@ -502,57 +499,35 @@ export class Client extends EventEmitter implements ProjectClientInterface { return synctable2.synctable(query, options, this, throttle_changes); } - // We leave in the project_id for consistency with the browser UI. - // And maybe someday we'll have tables managed across projects (?). - public async synctable_project(_project_id: string, query, _options) { - // TODO: this is ONLY for syncstring tables (syncstrings, patches, cursors). - // Also, options are ignored -- since we use whatever was selected by the frontend. - const the_synctable = await get_synctable(query, this); - // To provide same API, must also wait until done initializing. - if (the_synctable.get_state() !== "connected") { - await once(the_synctable, "connected"); - } - if (the_synctable.get_state() !== "connected") { - throw Error( - "Bug -- state of synctable must be connected " + JSON.stringify(query), - ); - } - return the_synctable; - } + conat = () => connectToConat(); - synctable_nats: NatsSyncTableFunction = async (query, options?) => { - return await synctable_nats(query, options); + synctable_conat: ConatSyncTableFunction = async (query, options?) => { + return await synctable_conat(query, options); }; - pubsub_nats = async ({ path, name }: { path?: string; name: string }) => { + pubsub_conat = async ({ path, name }: { path?: string; name: string }) => { return await pubsub({ path, name }); }; - callNatsService: CallNatsServiceFunction = async (options) => { - return await callNatsService(options); + callConatService: CallConatServiceFunction = async (options) => { + return await callConatService(options); }; - createNatsService: CreateNatsServiceFunction = (options) => { - return createNatsService({ + createConatService: CreateConatServiceFunction = (options) => { + return createConatService({ ...options, project_id: this.project_id, }); }; - getNatsEnv: NatsEnvFunction = async () => await getNatsEnv(); - // WARNING: making two of the exact same sync_string or sync_db will definitely // lead to corruption! // Get the synchronized doc with the given path. Returns undefined // if currently no such sync-doc. - public syncdoc({ path }: { path: string }): SyncDoc | undefined { - return get_syncdoc(path); - } - - public symmetric_channel(name) { - return symmetric_channel(name); - } + syncdoc = ({ path }: { path: string }): SyncDoc | undefined => { + return getSyncDoc(path); + }; public path_access(opts: { path: string; mode: string; cb: CB }): void { // mode: sub-sequence of 'rwxf' -- see https://nodejs.org/api/fs.html#fs_class_fs_stats @@ -655,29 +630,21 @@ export class Client extends EventEmitter implements ProjectClientInterface { // no-op; assumed async api touch_project(_project_id: string, _compute_server_id?: number) {} - async get_syncdoc_history(string_id: string, patches = false) { - const dbg = this.dbg("get_syncdoc_history"); - dbg(string_id, patches); - const mesg = message.get_syncdoc_history({ - string_id, - patches, - }); - return await callback2(this.call, { message: mesg }); - } - // Return true if the file was explicitly deleted. // Returns unknown if don't know // Returns false if definitely not. - public is_deleted(filename: string, _project_id: string) { - return !!getListingsTable()?.isDeleted(filename); + public is_deleted( + filename: string, + _project_id: string, + ): boolean | undefined { + return isDeleted(filename); } public async set_deleted( - filename: string, + _filename: string, _project_id?: string, ): Promise { - // project_id is ignored - const listings = getListingsTable(); - return await listings?.setDeleted(filename); + // DEPRECATED + this.dbg("set_deleted: DEPRECATED"); } } diff --git a/src/packages/project/nats/README.md b/src/packages/project/conat/README.md similarity index 64% rename from src/packages/project/nats/README.md rename to src/packages/project/conat/README.md index b3c001bf2d..f1cb3a8be2 100644 --- a/src/packages/project/nats/README.md +++ b/src/packages/project/conat/README.md @@ -1,9 +1,9 @@ -How to setup a standalone nodejs command line session to connect to nats **as a project** +How to setup a standalone nodejs command line session to connect to conat **as a project** -1. Create a file project-env.sh as explained in projects/nats/README.md, which defines these environment variables (your values will be different). You can use the command `export` from within a terminal in a project to find these values. +1. Create a file project-env.sh as explained in projects/conat/README.md, which defines these environment variables (your values will be different). You can use the command `export` from within a terminal in a project to find these values. ```sh -export NATS_SERVER="ws://localhost:5000/6b851643-360e-435e-b87e-f9a6ab64a8b1/port/5000/nats" +export CONAT_SERVER="http://localhost:5000/6b851643-360e-435e-b87e-f9a6ab64a8b1/port/5000" export COCALC_PROJECT_ID="00847397-d6a8-4cb0-96a8-6ef64ac3e6cf" export COCALC_USERNAME=`echo $COCALC_PROJECT_ID | tr -d '-'` export HOME="/projects/6b851643-360e-435e-b87e-f9a6ab64a8b1/cocalc/src/data/projects/$COCALC_PROJECT_ID" @@ -31,4 +31,4 @@ $ . project-env.sh $ node ``` -Now anything involving NATS will work with identity the project. +Now anything involving conat will work with identity the project. diff --git a/src/packages/project/conat/api/editor.ts b/src/packages/project/conat/api/editor.ts new file mode 100644 index 0000000000..32358e614d --- /dev/null +++ b/src/packages/project/conat/api/editor.ts @@ -0,0 +1,34 @@ +export { jupyter_strip_notebook as jupyterStripNotebook } from "@cocalc/jupyter/nbgrader/jupyter-parse"; +export { jupyter_run_notebook as jupyterRunNotebook } from "@cocalc/jupyter/nbgrader/jupyter-run"; +export { nbconvert as jupyterNbconvert } from "../../jupyter/convert"; +export { run_formatter_string as formatterString } from "../../formatters"; +export { logo as jupyterKernelLogo } from "@cocalc/jupyter/kernel/logo"; +export { get_kernel_data as jupyterKernels } from "@cocalc/jupyter/kernel/kernel-data"; +export { newFile } from "@cocalc/backend/misc/new-file"; + +import { printSageWS as printSageWS0 } from "@cocalc/project/print_to_pdf"; +import { filename_extension } from "@cocalc/util/misc"; +export async function printSageWS(opts): Promise { + let pdf; + const ext = filename_extension(opts.path); + if (ext) { + pdf = `${opts.path.slice(0, opts.path.length - ext.length)}pdf`; + } else { + pdf = opts.path + ".pdf"; + } + + await printSageWS0({ + path: opts.path, + outfile: pdf, + title: opts.options?.title, + author: opts.options?.author, + date: opts.options?.date, + contents: opts.options?.contents, + subdir: opts.options?.subdir, + extra_data: opts.options?.extra_data, + timeout: opts.options?.timeout, + }); + return pdf; +} + +export { createTerminalService } from "@cocalc/project/conat/terminal"; diff --git a/src/packages/project/conat/api/index.ts b/src/packages/project/conat/api/index.ts new file mode 100644 index 0000000000..7b2623900a --- /dev/null +++ b/src/packages/project/conat/api/index.ts @@ -0,0 +1,170 @@ +/* + +DEVELOPMENT: + +How to do development (so in a dev project doing cc-in-cc dev). + +0. From the browser, terminate this api server running in the project: + + await cc.client.conat_client.projectApi(cc.current()).system.terminate({service:'api'}) + +1. Create a file project-env.sh as explained in projects/conat/README.md, which defines these environment variables (your values will be different): + + export COCALC_PROJECT_ID="00847397-d6a8-4cb0-96a8-6ef64ac3e6cf" + export COCALC_USERNAME=`echo $COCALC_PROJECT_ID | tr -d '-'` + export HOME="/projects/6b851643-360e-435e-b87e-f9a6ab64a8b1/cocalc/src/data/projects/$COCALC_PROJECT_ID" + export DATA=$HOME/.smc + + # CRITICAL: make sure to create and set an api key! Otherwise you will be blocked: + export API_KEY=sk-OUwxAN8d0n7Ecd48000055 + export COMPUTE_SERVER_ID=0 + + # optional for more logging + export DEBUG=cocalc:* + export DEBUG_CONSOLE=yes + +If API_KEY is a project-wide API key, then you can change +COCALC_PROJECT_ID however you want and don't have to worry +about whether the project is running or the project secret +key changing when the project is restarted. + +2. Then do this: + + $ . project-env.sh + $ node + ... + > require("@cocalc/project/conat/api/index").init() + +You can then easily be able to grab some state, e.g., by writing this in any cocalc code, +rebuilding and restarting: + + global.x = {...} + +Remember, if you don't set API_KEY, then the project MUST be running so that the secret token in $HOME/.smc/secret_token is valid. + +3. Use the browser to see the project is on the conat network and works: + + a = cc.client.conat_client.projectApi({project_id:'81e0c408-ac65-4114-bad5-5f4b6539bd0e'}); + await a.system.ping(); + await a.system.exec({command:'echo $COCALC_PROJECT_ID'}); + +*/ + +import { type ProjectApi } from "@cocalc/conat/project/api"; +import { connectToConat } from "@cocalc/project/conat/connection"; +import { getSubject } from "../names"; +import { terminate as terminateOpenFiles } from "@cocalc/project/conat/open-files"; +import { close as closeListings } from "@cocalc/project/conat/listings"; +import { project_id } from "@cocalc/project/data"; +import { close as closeFilesRead } from "@cocalc/project/conat/files/read"; +import { close as closeFilesWrite } from "@cocalc/project/conat/files/write"; +import { getLogger } from "@cocalc/project/logger"; + +const logger = getLogger("conat:api"); + +export function init() { + serve(); +} + +let terminate = false; +async function serve() { + logger.debug("serve: create project conat api service"); + const cn = connectToConat(); + const subject = getSubject({ service: "api" }); + // @ts-ignore + const name = `project-${project_id}`; + logger.debug(`serve: creating api service ${name}`); + const api = await cn.subscribe(subject); + logger.debug(`serve: subscribed to subject='${subject}'`); + await listen(api, subject); +} + +async function listen(api, subject) { + for await (const mesg of api) { + if (terminate) { + return; + } + (async () => { + try { + await handleMessage(api, subject, mesg); + } catch (err) { + logger.debug(`WARNING: issue handling a message -- ${err}`); + } + })(); + } +} + +async function handleMessage(api, subject, mesg) { + const request = mesg.data ?? ({} as any); + // logger.debug("got message", request); + if (request.name == "system.terminate") { + // TODO: should be part of handleApiRequest below, but done differently because + // one case halts this loop + const { service } = request.args[0] ?? {}; + if (service == "open-files") { + terminateOpenFiles(); + mesg.respond({ status: "terminated", service }); + return; + } else if (service == "listings") { + closeListings(); + mesg.respond({ status: "terminated", service }); + return; + } else if (service == "files:read") { + await closeFilesRead(); + mesg.respond({ status: "terminated", service }); + return; + } else if (service == "files:write") { + await closeFilesWrite(); + mesg.respond({ status: "terminated", service }); + return; + } else if (service == "api") { + // special hook so admin can terminate handling. This is useful for development. + terminate = true; + console.warn("TERMINATING listening on ", subject); + logger.debug("TERMINATING listening on ", subject); + mesg.respond({ status: "terminated", service }); + api.stop(); + return; + } else { + mesg.respond({ error: `Unknown service ${service}` }); + } + } else { + handleApiRequest(request, mesg); + } +} + +async function handleApiRequest(request, mesg) { + let resp; + const { name, args } = request as any; + if (name == "ping") { + resp = "pong"; + } else { + try { + // logger.debug("handling project.api request:", { name }); + resp = (await getResponse({ name, args })) ?? null; + } catch (err) { + logger.debug(`project.api request err = ${err}`, { name }); + resp = { error: `${err}` }; + } + } + mesg.respond(resp); +} + +import * as system from "./system"; +import * as editor from "./editor"; +import * as sync from "./sync"; + +export const projectApi: ProjectApi = { + system, + editor, + sync, +}; + +async function getResponse({ name, args }) { + const [group, functionName] = name.split("."); + const f = projectApi[group]?.[functionName]; + if (f == null) { + throw Error(`unknown function '${name}'`); + } + return await f(...args); +} diff --git a/src/packages/project/nats/api/sync.ts b/src/packages/project/conat/api/sync.ts similarity index 100% rename from src/packages/project/nats/api/sync.ts rename to src/packages/project/conat/api/sync.ts diff --git a/src/packages/project/nats/api/system.ts b/src/packages/project/conat/api/system.ts similarity index 92% rename from src/packages/project/nats/api/system.ts rename to src/packages/project/conat/api/system.ts index cfe4febf12..60bedcc537 100644 --- a/src/packages/project/nats/api/system.ts +++ b/src/packages/project/conat/api/system.ts @@ -4,13 +4,6 @@ export async function ping() { export async function terminate() {} -import getConnection from "@cocalc/project/nats/connection"; -export async function resetConnection() { - const nc = await getConnection(); - await nc.close(); - return { closed: nc.isClosed() }; -} - import { handleExecShellCode } from "@cocalc/project/exec_shell_code"; export { handleExecShellCode as exec }; @@ -111,3 +104,6 @@ export async function signal({ throw errors[errors.length - 1]; } } + +import jupyterExecute from "@cocalc/jupyter/stateless-api/execute"; +export { jupyterExecute }; diff --git a/src/packages/project/nats/browser-websocket-api.ts b/src/packages/project/conat/browser-websocket-api.ts similarity index 57% rename from src/packages/project/nats/browser-websocket-api.ts rename to src/packages/project/conat/browser-websocket-api.ts index b6a14a0486..883d835d67 100644 --- a/src/packages/project/nats/browser-websocket-api.ts +++ b/src/packages/project/conat/browser-websocket-api.ts @@ -6,7 +6,7 @@ How to do development (so in a dev project doing cc-in-cc dev): 0. From the browser, send a terminate-handler message, so the handler running in the project stops: - await cc.client.nats_client.projectWebsocketApi({project_id:cc.current().project_id, mesg:{cmd:"terminate"}}) + await cc.client.conat_client.projectWebsocketApi({project_id:cc.current().project_id, mesg:{cmd:"terminate"}}) 1. Open a terminal in the project itself, which sets up the required environment variables. See api/index.ts for details!! @@ -14,11 +14,11 @@ How to do development (so in a dev project doing cc-in-cc dev): 3. Do this: - echo 'require("@cocalc/project/client").init(); require("@cocalc/project/nats/browser-websocket-api").init()' | DEBUG=cocalc:* DEBUG_CONSOLE=yes node + echo 'require("@cocalc/project/client").init(); require("@cocalc/project/conat/browser-websocket-api").init()' | DEBUG=cocalc:* DEBUG_CONSOLE=yes node Or just run node then paste in - require("@cocalc/project/client").init(); require("@cocalc/project/nats/browser-websocket-api").init() + require("@cocalc/project/client").init(); require("@cocalc/project/conat/browser-websocket-api").init() A nice thing about doing that is if you write this deep in some code: @@ -26,49 +26,46 @@ A nice thing about doing that is if you write this deep in some code: then after that code runs you can access x from the node console! -4. Use the browser to see the project is on nats and works: +4. Use the browser to see the project is on conat and works: - await cc.client.nats_client.projectWebsocketApi({project_id:'56eb622f-d398-489a-83ef-c09f1a1e8094', mesg:{cmd:"listing"}}) + await cc.client.conat_client.projectWebsocketApi({project_id:'56eb622f-d398-489a-83ef-c09f1a1e8094', mesg:{cmd:"listing"}}) -5. In a terminal you can always tap into the message stream for a particular project (do `pnpm nats-shell` if necessary to setup your environment): +5. In a terminal you can always tap into the message stream for a particular project: - nats sub --match-replies project.56eb622f-d398-489a-83ef-c09f1a1e8094.browser-api + cd packages/backend + pnpm conat-watch project.56eb622f-d398-489a-83ef-c09f1a1e8094.browser-api --match-replies */ import { getLogger } from "@cocalc/project/logger"; -import { JSONCodec } from "nats"; -import getConnection from "./connection"; +import { connectToConat } from "./connection"; import { handleApiCall } from "@cocalc/project/browser-websocket/api"; -import { getPrimusConnection } from "@cocalc/nats/primus"; import { getSubject } from "./names"; -const logger = getLogger("project:nats:browser-websocket-api"); - -const jc = JSONCodec(); +const logger = getLogger("project:conat:browser-websocket-api"); export async function init() { - const nc = await getConnection(); + const client = connectToConat(); const subject = getSubject({ service: "browser-api", }); - logger.debug(`initAPI -- NATS project subject '${subject}'`); - const sub = nc.subscribe(subject); - const primus = getPrimusConnection({ - subject: getSubject({ - service: "primus", - }), - env: { nc, jc }, - role: "server", - id: "project", + logger.debug(`initAPI -- project subject '${subject}'`); + const sub = await client.subscribe(subject); + logger.debug(`browser primus subject: ${getSubject({ service: "primus" })}`); + const primus = client.socket.listen(getSubject({ service: "primus" })); + primus.on("connection", (spark) => { + logger.debug("got a spark"); + spark.on("data", (data) => { + spark.write(`${data}`.repeat(3)); + }); }); for await (const mesg of sub) { - const data = jc.decode(mesg.data) ?? ({} as any); + const data = mesg.data ?? ({} as any); if (data.cmd == "terminate") { logger.debug( "received terminate-handler, so will not handle any further messages", ); - mesg.respond(jc.encode({ exiting: true })); + mesg.respond({ exiting: true }); return; } handleRequest({ data, mesg, primus }); @@ -84,5 +81,5 @@ async function handleRequest({ data, mesg, primus }) { resp = { error: `${err}` }; } //logger.debug("responded", resp); - mesg.respond(jc.encode(resp)); + mesg.respond(resp ?? null); } diff --git a/src/packages/project/conat/connection.ts b/src/packages/project/conat/connection.ts new file mode 100644 index 0000000000..1b33a456b8 --- /dev/null +++ b/src/packages/project/conat/connection.ts @@ -0,0 +1,105 @@ +/* +Create a connection to a conat server authenticated as a project or compute +server, via an api key or the project secret token. +*/ + +import { apiKey, conatServer } from "@cocalc/backend/data"; +import { secretToken } from "@cocalc/project/data"; +import { connect, type Client } from "@cocalc/conat/core/client"; +import { + API_COOKIE_NAME, + PROJECT_SECRET_COOKIE_NAME, + PROJECT_ID_COOKIE_NAME, +} from "@cocalc/backend/auth/cookie-names"; +import { inboxPrefix } from "@cocalc/conat/names"; +import { setConatClient } from "@cocalc/conat/client"; +import { compute_server_id, project_id } from "@cocalc/project/data"; +import { version as ourVersion } from "@cocalc/util/smc-version"; +import { getLogger } from "@cocalc/project/logger"; +import { initHubApi } from "@cocalc/conat/hub/api"; +import { delay } from "awaiting"; + +const logger = getLogger("conat:connection"); + +const VERSION_CHECK_INTERVAL = 2 * 60000; + +let cache: Client | null = null; +export function connectToConat(options?): Client { + if (cache != null) { + return cache; + } + let Cookie; + if (apiKey) { + Cookie = `${API_COOKIE_NAME}=${apiKey}`; + } else { + Cookie = `${PROJECT_SECRET_COOKIE_NAME}=${secretToken}; ${PROJECT_ID_COOKIE_NAME}=${project_id}`; + } + cache = connect({ + address: conatServer, + inboxPrefix: inboxPrefix({ project_id }), + extraHeaders: { Cookie }, + ...options, + }); + + versionCheckLoop(cache); + + return cache!; +} + +export function init() { + setConatClient({ + conat: connectToConat, + project_id, + compute_server_id, + getLogger, + }); +} +init(); + +async function callHub({ + client, + service = "api", + name, + args = [], + timeout, +}: { + client: Client; + service?: string; + name: string; + args: any[]; + timeout?: number; +}) { + const subject = `hub.project.${project_id}.${service}`; + try { + const data = { name, args }; + const resp = await client.request(subject, data, { timeout }); + return resp.data; + } catch (err) { + err.message = `${err.message} - callHub: subject='${subject}', name='${name}', `; + throw err; + } +} + +async function versionCheckLoop(client) { + const hub = initHubApi((opts) => callHub({ ...opts, client })); + while (true) { + try { + const { version } = await hub.system.getCustomize(["version"]); + logger.debug("versionCheckLoop: ", { ...version, ourVersion }); + if (version != null) { + const requiredVersion = compute_server_id + ? (version.min_compute_server ?? 0) + : (version.min_project ?? 0); + if ((ourVersion ?? 0) < requiredVersion) { + logger.debug( + `ERROR: our CoCalc version ${ourVersion} is older than the required version ${requiredVersion}. \n\n** TERMINATING DUE TO VERSION BEING TOO OLD!!**\n\n`, + ); + setTimeout(() => process.exit(1), 10); + } + } + } catch (err) { + logger.debug(`WARNING: problem getting version info from hub -- ${err}`); + } + await delay(VERSION_CHECK_INTERVAL); + } +} diff --git a/src/packages/project/conat/env.ts b/src/packages/project/conat/env.ts new file mode 100644 index 0000000000..411e22e2a7 --- /dev/null +++ b/src/packages/project/conat/env.ts @@ -0,0 +1,14 @@ +import { connectToConat } from "./connection"; +import { setConatClient } from "@cocalc/conat/client"; +import { compute_server_id, project_id } from "@cocalc/project/data"; +import { getLogger } from "@cocalc/project/logger"; + +export function init() { + setConatClient({ + conat: () => connectToConat(), + project_id, + compute_server_id, + getLogger, + }); +} +init(); diff --git a/src/packages/project/nats/files/read.ts b/src/packages/project/conat/files/read.ts similarity index 76% rename from src/packages/project/nats/files/read.ts rename to src/packages/project/conat/files/read.ts index 76882f2fd0..e14701f8a5 100644 --- a/src/packages/project/nats/files/read.ts +++ b/src/packages/project/conat/files/read.ts @@ -5,7 +5,7 @@ DEVELOPMENT: 1. Stop files:read service running in the project by running this in your browser: - await cc.client.nats_client.projectApi(cc.current()).system.terminate({service:'files:read'}) + await cc.client.conat_client.projectApi(cc.current()).system.terminate({service:'files:read'}) {status: 'terminated', service: 'files:read'} @@ -14,17 +14,17 @@ You can also skip step 1 if you instead set COMPUTE_SERVER_ID to something nonze 2. Setup the project environment variables. Then start the server in node: - ~/cocalc/src/packages/project/nats$ . project-env.sh + ~/cocalc/src/packages/project/conat$ . project-env.sh $ node Welcome to Node.js v18.17.1. Type ".help" for more information. - require('@cocalc/project/nats/files/read').init() + require('@cocalc/project/conat/files/read').init() */ -import "@cocalc/project/nats/env"; // ensure nats env available +import "@cocalc/project/conat/env"; // ensure conat env available import { createReadStream as fs_createReadStream } from "fs"; import { compute_server_id, project_id } from "@cocalc/project/data"; @@ -32,7 +32,7 @@ import { join } from "path"; import { createServer, close as closeReadServer, -} from "@cocalc/nats/files/read"; +} from "@cocalc/conat/files/read"; function createReadStream(path: string) { if (path[0] != "/" && process.env.HOME) { diff --git a/src/packages/project/nats/files/write.ts b/src/packages/project/conat/files/write.ts similarity index 82% rename from src/packages/project/nats/files/write.ts rename to src/packages/project/conat/files/write.ts index 576618c0f8..1fb55ae085 100644 --- a/src/packages/project/nats/files/write.ts +++ b/src/packages/project/conat/files/write.ts @@ -5,7 +5,7 @@ DEVELOPMENT: 1. Stop the files:write service running in the project by running this in your browser: - await cc.client.nats_client.projectApi(cc.current()).system.terminate({service:'files:write'}) + await cc.client.conat_client.projectApi(cc.current()).system.terminate({service:'files:write'}) {status: 'terminated', service: 'files:write'} @@ -14,17 +14,17 @@ You can also skip step 1 if you instead set COMPUTE_SERVER_ID to something nonze 2. Setup the project environment variables. Then start the server in node: - ~/cocalc/src/packages/project/nats$ . project-env.sh + ~/cocalc/src/packages/project/conat$ . project-env.sh $ node Welcome to Node.js v18.17.1. Type ".help" for more information. - require('@cocalc/project/nats/files/write').init() + require('@cocalc/project/conat/files/write').init() */ -import "@cocalc/project/nats/env"; // ensure nats env available +import "@cocalc/project/conat/env"; // ensure conat env available import ensureContainingDirectoryExists from "@cocalc/backend/misc/ensure-containing-directory-exists"; import { createWriteStream as fs_createWriteStream } from "fs"; import { rename } from "fs/promises"; @@ -33,8 +33,8 @@ import { join } from "path"; import { createServer, close as closeWriteServer, -} from "@cocalc/nats/files/write"; -import { randomId } from "@cocalc/nats/names"; +} from "@cocalc/conat/files/write"; +import { randomId } from "@cocalc/conat/names"; import { rimraf } from "rimraf"; async function createWriteStream(path: string) { diff --git a/src/packages/project/nats/formatter.ts b/src/packages/project/conat/formatter.ts similarity index 88% rename from src/packages/project/nats/formatter.ts rename to src/packages/project/conat/formatter.ts index 8381c4f61d..aa593e89bc 100644 --- a/src/packages/project/nats/formatter.ts +++ b/src/packages/project/conat/formatter.ts @@ -3,7 +3,7 @@ File formatting service. */ import { run_formatter, type Options } from "../formatters"; -import { createFormatterService as create } from "@cocalc/nats/service/formatter"; +import { createFormatterService as create } from "@cocalc/conat/service/formatter"; import { compute_server_id, project_id } from "@cocalc/project/data"; interface Message { diff --git a/src/packages/project/nats/index.ts b/src/packages/project/conat/index.ts similarity index 68% rename from src/packages/project/nats/index.ts rename to src/packages/project/conat/index.ts index d3167ac579..056c9c8c7b 100644 --- a/src/packages/project/nats/index.ts +++ b/src/packages/project/conat/index.ts @@ -6,6 +6,7 @@ Start the NATS servers: - websocket api (temporary/legacy shim) */ +import "./connection"; import { getLogger } from "@cocalc/project/logger"; import { init as initAPI } from "./api"; import { init as initOpenFiles } from "./open-files"; @@ -14,15 +15,19 @@ import { init as initWebsocketApi } from "./browser-websocket-api"; import { init as initListings } from "./listings"; import { init as initRead } from "./files/read"; import { init as initWrite } from "./files/write"; +import { init as initProjectStatus } from "@cocalc/project/project-status/server"; +import { init as initUsageInfo } from "@cocalc/project/usage-info"; -const logger = getLogger("project:nats:index"); +const logger = getLogger("project:conat:index"); export default async function init() { - logger.debug("starting NATS project services"); + logger.debug("starting Conat project services"); await initAPI(); await initOpenFiles(); initWebsocketApi(); await initListings(); await initRead(); await initWrite(); + initProjectStatus(); + initUsageInfo(); } diff --git a/src/packages/project/nats/listings.ts b/src/packages/project/conat/listings.ts similarity index 84% rename from src/packages/project/nats/listings.ts rename to src/packages/project/conat/listings.ts index d8dbda7b39..513e66dc37 100644 --- a/src/packages/project/nats/listings.ts +++ b/src/packages/project/conat/listings.ts @@ -13,7 +13,7 @@ DEVELOPMENT: 1. Stop listings service running in the project by running this in your browser: - await cc.client.nats_client.projectApi(cc.current()).system.terminate({service:'listings'}) + await cc.client.conat_client.projectApi(cc.current()).system.terminate({service:'listings'}) {status: 'terminated', service: 'listings'} @@ -22,10 +22,10 @@ DEVELOPMENT: 3. Start your own server -.../src/packages/project/nats$ node +.../src/packages/project/conat$ node - await require('@cocalc/project/nats/listings').init() + await require('@cocalc/project/conat/listings').init() */ @@ -38,18 +38,19 @@ import { INTEREST_CUTOFF_MS, type Listing, type Times, -} from "@cocalc/nats/service/listings"; +} from "@cocalc/conat/service/listings"; import { compute_server_id, project_id } from "@cocalc/project/data"; import { init as initClient } from "@cocalc/project/client"; import { delay } from "awaiting"; import { type DKV } from "./sync"; -import { type NatsService } from "@cocalc/nats/service"; +import { type ConatService } from "@cocalc/conat/service"; import { MultipathWatcher } from "@cocalc/backend/path-watcher"; import getLogger from "@cocalc/backend/logger"; +import { path_split } from "@cocalc/util/misc"; -const logger = getLogger("project:nats:listings"); +const logger = getLogger("project:conat:listings"); -let service: NatsService | null; +let service: ConatService | null; export async function init() { logger.debug("init: initializing"); initClient(); @@ -86,6 +87,10 @@ const impl = { }, }; +export function isDeleted(filename: string) { + return listings?.isDeleted(filename); +} + class Listings { private listings: DKV; @@ -189,6 +194,21 @@ class Listings { this.times.set(path, { ...this.times.get(path), interest: Date.now() }); this.updateListing(path); }; + + isDeleted = (filename: string): boolean | undefined => { + if (this.listings == null) { + return undefined; + } + const { head: path, tail } = path_split(filename); + const listing = this.listings.get(path); + if (listing == null) { + return undefined; + } + if (listing.deleted?.includes(tail)) { + return true; + } + return false; + }; } // this does a tiny amount to make paths more canonical. diff --git a/src/packages/project/nats/names.ts b/src/packages/project/conat/names.ts similarity index 81% rename from src/packages/project/nats/names.ts rename to src/packages/project/conat/names.ts index 9b1dbc1301..0b2611dba2 100644 --- a/src/packages/project/nats/names.ts +++ b/src/packages/project/conat/names.ts @@ -1,5 +1,5 @@ import { compute_server_id, project_id } from "@cocalc/project/data"; -import { projectSubject, projectStreamName } from "@cocalc/nats/names"; +import { projectSubject, projectStreamName } from "@cocalc/conat/names"; export function getSubject(opts: { path?: string; service: string }) { return projectSubject({ ...opts, compute_server_id, project_id }); diff --git a/src/packages/project/nats/open-files.ts b/src/packages/project/conat/open-files.ts similarity index 76% rename from src/packages/project/nats/open-files.ts rename to src/packages/project/conat/open-files.ts index 7bc84c6f84..10c96d1ede 100644 --- a/src/packages/project/nats/open-files.ts +++ b/src/packages/project/conat/open-files.ts @@ -5,20 +5,19 @@ DEVELOPMENT: 0. From the browser with the project opened, terminate the open-files api service: - await cc.client.nats_client.projectApi(cc.current()).system.terminate({service:'open-files'}) + await cc.client.conat_client.projectApi(cc.current()).system.terminate({service:'open-files'}) - // {status: 'terminated', service: 'open-files'} Set env variables as in a project (see api/index.ts ), then in nodejs: -DEBUG_CONSOLE=yes DEBUG=cocalc:debug:project:nats:* node +DEBUG_CONSOLE=yes DEBUG=cocalc:debug:project:conat:* node - x = await require("@cocalc/project/nats/open-files").init(); Object.keys(x) + x = await require("@cocalc/project/conat/open-files").init(); Object.keys(x) -[ 'openFiles', 'openDocs', 'formatter', 'terminate', 'computeServers' ] +[ 'openFiles', 'openDocs', 'formatter', 'terminate', 'computeServers', 'cc' ] > x.openFiles.getAll(); @@ -32,7 +31,7 @@ DEBUG_CONSOLE=yes DEBUG=cocalc:debug:project:nats:* node OR: - echo "require('@cocalc/project/nats/open-files').init(); require('@cocalc/project/bug-counter').init()" | node + echo "require('@cocalc/project/conat/open-files').init(); require('@cocalc/project/bug-counter').init()" | node COMPUTE SERVER: @@ -64,10 +63,10 @@ Ssh in to the project itself. You can use a terminal because that very terminal doing this! Then: /cocalc/github/src/packages/project$ . /cocalc/nvm/nvm.sh -/cocalc/github/src/packages/project$ COCALC_PROJECT_ID=... COCALC_SECRET_TOKEN="/secrets/secret-token/token" NATS_SERVER=nats-server node +/cocalc/github/src/packages/project$ COCALC_PROJECT_ID=... COCALC_SECRET_TOKEN="/secrets/secret-token/token" CONAT_SERVER=hub-conat node # not sure about CONAT_SERVER Welcome to Node.js v20.19.0. Type ".help" for more information. -> x = await require("@cocalc/project/nats/open-files").init(); Object.keys(x) +> x = await require("@cocalc/project/conat/open-files").init(); Object.keys(x) [ 'openFiles', 'openDocs', 'formatter', 'terminate', 'computeServers' ] > @@ -78,9 +77,8 @@ import { openFiles as createOpenFiles, type OpenFiles, type OpenFileEntry, -} from "@cocalc/project/nats/sync"; -import { getSyncDocType } from "@cocalc/nats/sync/syncdoc-info"; -import { NATS_OPEN_FILE_TOUCH_INTERVAL } from "@cocalc/util/nats"; +} from "@cocalc/project/conat/sync"; +import { CONAT_OPEN_FILE_TOUCH_INTERVAL } from "@cocalc/util/conat"; import { compute_server_id, project_id } from "@cocalc/project/data"; import type { SyncDoc } from "@cocalc/sync/editor/generic/sync-doc"; import { getClient } from "@cocalc/project/client"; @@ -92,8 +90,7 @@ import { delay } from "awaiting"; import { initJupyterRedux, removeJupyterRedux } from "@cocalc/jupyter/kernel"; import { filename_extension, original_path } from "@cocalc/util/misc"; import { createFormatterService } from "./formatter"; -import { type NatsService } from "@cocalc/nats/service/service"; -import { createTerminalService } from "./terminal"; +import { type ConatService } from "@cocalc/conat/service/service"; import { exists } from "@cocalc/backend/misc/async-utils-node"; import { map as awaitMap } from "awaiting"; import { unlink } from "fs/promises"; @@ -101,14 +98,15 @@ import { join } from "path"; import { computeServerManager, ComputeServerManager, -} from "@cocalc/nats/compute/manager"; +} from "@cocalc/conat/compute/manager"; import { JUPYTER_SYNCDB_EXTENSIONS } from "@cocalc/util/jupyter/names"; +import { connectToConat } from "@cocalc/project/conat/connection"; -// ensure nats connection stuff is initialized -import "@cocalc/project/nats/env"; +// ensure conat connection stuff is initialized +import "@cocalc/project/conat/env"; import { chdir } from "node:process"; -const logger = getLogger("project:nats:open-files"); +const logger = getLogger("project:conat:open-files"); // we check all files we are currently managing this frequently to // see if they exist on the filesystem: @@ -127,10 +125,18 @@ const FILE_DELETION_INITIAL_DELAY = 15000; let openFiles: OpenFiles | null = null; let formatter: any = null; -const openDocs: { [path: string]: SyncDoc | NatsService } = {}; +const openDocs: { [path: string]: SyncDoc | ConatService } = {}; let computeServers: ComputeServerManager | null = null; const openTimes: { [path: string]: number } = {}; +export function getSyncDoc(path: string): SyncDoc | undefined { + const doc = openDocs[path]; + if (doc instanceof SyncString || doc instanceof SyncDB) { + return doc; + } + return undefined; +} + export async function init() { logger.debug("init"); @@ -141,7 +147,7 @@ export async function init() { openFiles = await createOpenFiles(); computeServers = computeServerManager({ project_id }); - await computeServers.init(); + await computeServers.waitUntilReady(); computeServers.on("change", async ({ path, id }) => { if (openFiles == null) { return; @@ -170,13 +176,26 @@ export async function init() { // handle changes openFiles.on("change", (entry) => { - handleChange(entry); + // we ONLY actually try to open the file here if there + // is a doctype set. When it is first being created, + // the doctype won't be the first field set, and we don't + // want to launch this until it is set. + if (entry.doctype) { + handleChange(entry); + } }); formatter = await createFormatterService({ openSyncDocs: openDocs }); // useful for development - return { openFiles, openDocs, formatter, terminate, computeServers }; + return { + openFiles, + openDocs, + formatter, + terminate, + computeServers, + cc: connectToConat(), + }; } export function terminate() { @@ -195,7 +214,7 @@ export function terminate() { } function getCutoff(): number { - return Date.now() - 2.5 * NATS_OPEN_FILE_TOUCH_INTERVAL; + return Date.now() - 2.5 * CONAT_OPEN_FILE_TOUCH_INTERVAL; } function computeServerId(path: string): number { @@ -207,46 +226,52 @@ async function handleChange({ time, deleted, backend, + doctype, id, }: OpenFileEntry & { id?: number }) { - if (id == null) { - id = computeServerId(path); - } - logger.debug("handleChange", { path, time, deleted, backend, id }); - const syncDoc = openDocs[path]; - const isOpenHere = syncDoc != null; - - if (id != compute_server_id) { - if (backend?.id == compute_server_id) { - // we are definitely not the backend right now. - openFiles?.setNotBackend(path, compute_server_id); - } - // only thing we should do is close it if it is open. - if (isOpenHere) { - await closeDoc(path); + try { + if (id == null) { + id = computeServerId(path); } - return; - } - - if (deleted?.deleted) { - if (await exists(path)) { - // it's back - openFiles?.setNotDeleted(path); - } else { + logger.debug("handleChange", { path, time, deleted, backend, doctype, id }); + const syncDoc = openDocs[path]; + const isOpenHere = syncDoc != null; + + if (id != compute_server_id) { + if (backend?.id == compute_server_id) { + // we are definitely not the backend right now. + openFiles?.setNotBackend(path, compute_server_id); + } + // only thing we should do is close it if it is open. if (isOpenHere) { await closeDoc(path); } return; } - } - if (time != null && time >= getCutoff()) { - if (!isOpenHere) { - logger.debug("handleChange: opening", { path }); - // users actively care about this file being opened HERE, but it isn't - await openDoc(path); + if (deleted?.deleted) { + if (await exists(path)) { + // it's back + openFiles?.setNotDeleted(path); + } else { + if (isOpenHere) { + await closeDoc(path); + } + return; + } } - return; + + if (time != null && time >= getCutoff()) { + if (!isOpenHere) { + logger.debug("handleChange: opening", { path }); + // users actively care about this file being opened HERE, but it isn't + await openDoc(path); + } + return; + } + } catch (err) { + console.trace(err); + logger.debug(`handleChange: WARNING - error opening ${path} -- ${err}`); } } @@ -265,7 +290,7 @@ function supportAutoclose(path: string): boolean { async function closeIgnoredFilesLoop() { while (openFiles?.state == "connected") { - await delay(NATS_OPEN_FILE_TOUCH_INTERVAL); + await delay(CONAT_OPEN_FILE_TOUCH_INTERVAL); if (openFiles?.state != "connected") { return; } @@ -300,7 +325,7 @@ async function touchOpenFilesLoop() { for (const path in openDocs) { openFiles.setBackend(path, compute_server_id); } - await delay(NATS_OPEN_FILE_TOUCH_INTERVAL); + await delay(CONAT_OPEN_FILE_TOUCH_INTERVAL); } } @@ -416,21 +441,31 @@ const openDoc = reuseInFlight(async (path: string) => { if (doc != null) { return; } - openTimes[path] = Date.now(); + openTimes[path] = Date.now(); if (path.endsWith(".term")) { - const service = await createTerminalService(path); - openDocs[path] = service; + // terminals are handled directly by the project api -- also since + // doctype probably not set for them, they won't end up here. + // (this could change though, e.g., we might use doctype to + // set the terminal command). return; } const client = getClient(); - const doctype = await getSyncDocType({ - project_id, + let doctype: any = openFiles?.get(path)?.doctype; + logger.debug("openDoc: open files table knows ", openFiles?.get(path), { path, - client, }); - logger.debug("openDoc got", { path, doctype }); + if (doctype == null) { + logger.debug("openDoc: doctype must be set but isn't, so bailing", { + path, + }); + } else { + logger.debug("openDoc: got doctype from openFiles table", { + path, + doctype, + }); + } let syncdoc; if (doctype.type == "string") { diff --git a/src/packages/project/conat/pubsub.ts b/src/packages/project/conat/pubsub.ts new file mode 100644 index 0000000000..d922dcce42 --- /dev/null +++ b/src/packages/project/conat/pubsub.ts @@ -0,0 +1,13 @@ +import { connectToConat } from "./connection"; +import { PubSub } from "@cocalc/conat/sync/pubsub"; +import { project_id } from "@cocalc/project/data"; + +export default function pubsub({ + path, + name, +}: { + path?: string; + name: string; +}) { + return new PubSub({ client: connectToConat(), project_id, path, name }); +} diff --git a/src/packages/project/conat/sync.ts b/src/packages/project/conat/sync.ts new file mode 100644 index 0000000000..ca2c343501 --- /dev/null +++ b/src/packages/project/conat/sync.ts @@ -0,0 +1,59 @@ +import { + dstream as createDstream, + type DStream, + type DStreamOptions, +} from "@cocalc/conat/sync/dstream"; +import { + dkv as createDKV, + type DKV, + type DKVOptions, +} from "@cocalc/conat/sync/dkv"; +import { dko as createDKO, type DKO } from "@cocalc/conat/sync/dko"; +import { project_id } from "@cocalc/project/data"; +import { + createOpenFiles, + type OpenFiles, + Entry as OpenFileEntry, +} from "@cocalc/conat/sync/open-files"; +import { + inventory as createInventory, + type Inventory, +} from "@cocalc/conat/sync/inventory"; + +import { akv as createAKV, type AKV } from "@cocalc/conat/sync/akv"; +import { + astream as createAStream, + type AStream, +} from "@cocalc/conat/sync/astream"; + +export type { DStream, DKV, OpenFiles, OpenFileEntry }; + +export async function dstream( + opts: DStreamOptions, +): Promise> { + return await createDstream({ project_id, ...opts }); +} + +export async function dkv(opts: DKVOptions): Promise> { + return await createDKV({ project_id, ...opts }); +} + +export function akv(opts: DKVOptions): AKV { + return createAKV({ project_id, ...opts }); +} + +export function astream(opts: DStreamOptions): AStream { + return createAStream({ project_id, ...opts }); +} + +export async function dko(opts: DKVOptions): Promise> { + return await createDKO({ project_id, ...opts }); +} + +export async function openFiles(): Promise { + return await createOpenFiles({ project_id }); +} + +export async function inventory(): Promise { + return await createInventory({ project_id }); +} diff --git a/src/packages/project/conat/synctable.ts b/src/packages/project/conat/synctable.ts new file mode 100644 index 0000000000..b7c36ee6be --- /dev/null +++ b/src/packages/project/conat/synctable.ts @@ -0,0 +1,23 @@ +import { connectToConat } from "./connection"; +import { project_id } from "@cocalc/project/data"; +import { + type ConatSyncTable, + type ConatSyncTableFunction, +} from "@cocalc/conat/sync/synctable"; +import { parseQueryWithOptions } from "@cocalc/sync/table/util"; + +const synctable: ConatSyncTableFunction = async ( + query0, + options?, +): Promise => { + const { query, table } = parseQueryWithOptions(query0, options); + const client = connectToConat(); + query[table][0].project_id = project_id; + return await client.sync.synctable({ + project_id, + ...options, + query, + }); +}; + +export default synctable; diff --git a/src/packages/project/conat/terminal/index.ts b/src/packages/project/conat/terminal/index.ts new file mode 100644 index 0000000000..d22dc533d1 --- /dev/null +++ b/src/packages/project/conat/terminal/index.ts @@ -0,0 +1,5 @@ +/* +Terminal +*/ + +export { createTerminalService } from "./manager"; diff --git a/src/packages/project/conat/terminal/manager.ts b/src/packages/project/conat/terminal/manager.ts new file mode 100644 index 0000000000..d93b99093d --- /dev/null +++ b/src/packages/project/conat/terminal/manager.ts @@ -0,0 +1,238 @@ +import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; +import { getLogger } from "@cocalc/project/logger"; +import { + createTerminalServer, + type ConatService, +} from "@cocalc/conat/service/terminal"; +import { project_id, compute_server_id } from "@cocalc/project/data"; +import { isEqual } from "lodash"; +import ensureContainingDirectoryExists from "@cocalc/backend/misc/ensure-containing-directory-exists"; +import { Session } from "./session"; +import { + computeServerManager, + ComputeServerManager, +} from "@cocalc/conat/compute/manager"; +const logger = getLogger("project:conat:terminal:manager"); +import type { CreateTerminalOptions } from "@cocalc/conat/project/api/editor"; + +let manager: TerminalManager | null = null; +export const createTerminalService = async ( + termPath: string, + opts?: CreateTerminalOptions, +) => { + if (manager == null) { + logger.debug("createTerminalService -- creating manager"); + manager = new TerminalManager(); + } + return await manager.createTerminalService(termPath, opts); +}; + +export function pidToPath(pid: number): string | undefined { + return manager?.pidToPath(pid); +} + +export class TerminalManager { + private services: { [termPath: string]: ConatService } = {}; + private sessions: { [termPath: string]: Session } = {}; + private computeServers?: ComputeServerManager; + + constructor() { + this.computeServers = computeServerManager({ project_id }); + this.computeServers.on("change", this.handleComputeServersChange); + } + + private handleComputeServersChange = async ({ path: termPath, id = 0 }) => { + const service = this.services[termPath]; + if (service == null) return; + if (id != compute_server_id) { + logger.debug( + `terminal '${termPath}' moved: ${compute_server_id} --> ${id}: Stopping`, + ); + this.sessions[termPath]?.close(); + service.close(); + delete this.services[termPath]; + delete this.sessions[termPath]; + } + }; + + close = () => { + logger.debug("close"); + if (this.computeServers == null) { + return; + } + for (const termPath in this.services) { + this.services[termPath].close(); + } + this.services = {}; + this.sessions = {}; + this.computeServers.removeListener( + "change", + this.handleComputeServersChange, + ); + this.computeServers.close(); + delete this.computeServers; + }; + + private getSession = async ( + termPath: string, + options, + noCreate?: boolean, + ): Promise => { + const cur = this.sessions[termPath]; + if (cur != null) { + return cur; + } + if (noCreate) { + throw Error("no terminal session"); + } + await this.createTerminal({ ...options, termPath }); + const session = this.sessions[termPath]; + if (session == null) { + throw Error( + `BUG: failed to create terminal session - ${termPath} (this should not happen)`, + ); + } + return session; + }; + + createTerminalService = reuseInFlight( + async (termPath: string, opts?: CreateTerminalOptions) => { + if (this.services[termPath] != null) { + return; + } + let options: any = undefined; + + const getSession = async (options, noCreate?) => + await this.getSession(termPath, options, noCreate); + + const impl = { + create: async ( + opts: CreateTerminalOptions, + ): Promise<{ success: "ok"; note?: string; ephemeral?: boolean }> => { + // save options to reuse. + options = opts; + const note = await this.createTerminal({ ...opts, termPath }); + return { success: "ok", note }; + }, + + write: async (data: string): Promise => { + logger.debug("received data", data.length); + if (typeof data != "string") { + throw Error(`data must be a string -- ${JSON.stringify(data)}`); + } + const session = await getSession(options); + await session.write(data); + }, + + restart: async () => { + const session = await getSession(options); + await session.restart(); + }, + + cwd: async () => { + const session = await getSession(options); + return await session.getCwd(); + }, + + kill: async () => { + try { + const session = await getSession(options, true); + await session.close(); + } catch { + return; + } + }, + + size: async (opts: { + rows: number; + cols: number; + browser_id: string; + kick?: boolean; + }) => { + const session = await getSession(options); + session.setSize(opts); + }, + + close: async (browser_id: string) => { + this.sessions[termPath]?.browserLeaving(browser_id); + }, + }; + + const server = createTerminalServer({ termPath, project_id, impl }); + + server.on("close", () => { + this.sessions[termPath]?.close(); + delete this.sessions[termPath]; + delete this.services[termPath]; + }); + + this.services[termPath] = server; + + if (opts != null) { + await impl.create(opts); + } + }, + ); + + closeTerminal = (termPath: string) => { + const cur = this.sessions[termPath]; + if (cur != null) { + cur.close(); + delete this.sessions[termPath]; + } + }; + + createTerminal = reuseInFlight( + async (params) => { + if (params == null) { + throw Error("params must be specified"); + } + const { termPath, ...options } = params; + if (!termPath) { + throw Error("termPath must be specified"); + } + await ensureContainingDirectoryExists(termPath); + let note = ""; + const cur = this.sessions[termPath]; + if (cur != null) { + if (!isEqual(cur.options, options) || cur.state == "closed") { + // clean up -- we will make new one below + this.closeTerminal(termPath); + note += "Closed existing session. "; + } else { + // already have a working session with correct options + note += "Already have working session with same options. "; + return note; + } + } + note += "Creating new session."; + let session = new Session({ termPath, options }); + await session.init(); + if (session.state == "closed") { + // closed during init -- unlikely but possible; try one more time + session = new Session({ termPath, options }); + await session.init(); + if (session.state == "closed") { + throw Error(`unable to create terminal session for ${termPath}`); + } + } else { + this.sessions[termPath] = session; + return note; + } + }, + { + createKey: (args) => { + return args[0]?.termPath ?? ""; + }, + }, + ); + + pidToPath = (pid: number): string | undefined => { + for (const termPath in this.sessions) { + const s = this.sessions[termPath]; + if (s.pid == pid) { + return s.options.path; + } + } + }; +} diff --git a/src/packages/project/nats/terminal.ts b/src/packages/project/conat/terminal/session.ts similarity index 59% rename from src/packages/project/nats/terminal.ts rename to src/packages/project/conat/terminal/session.ts index 29edcca2ba..ad97bfb987 100644 --- a/src/packages/project/nats/terminal.ts +++ b/src/packages/project/conat/terminal/session.ts @@ -1,204 +1,69 @@ -/* -Terminal - -- using NATS -*/ - import { spawn } from "@lydell/node-pty"; import { envForSpawn } from "@cocalc/backend/misc"; import { path_split } from "@cocalc/util/misc"; import { console_init_filename, len } from "@cocalc/util/misc"; import { exists } from "@cocalc/backend/misc/async-utils-node"; -import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { getLogger } from "@cocalc/project/logger"; import { readlink, realpath } from "node:fs/promises"; -import { dstream, type DStream } from "@cocalc/project/nats/sync"; +import { dstream, type DStream } from "@cocalc/project/conat/sync"; import { - createTerminalServer, createBrowserClient, SIZE_TIMEOUT_MS, -} from "@cocalc/nats/service/terminal"; +} from "@cocalc/conat/service/terminal"; import { project_id, compute_server_id } from "@cocalc/project/data"; -import { isEqual, throttle } from "lodash"; +import { throttle } from "lodash"; +import { ThrottleString as Throttle } from "@cocalc/util/throttle"; +import { join } from "path"; +import type { CreateTerminalOptions } from "@cocalc/conat/project/api/editor"; +import { delay } from "awaiting"; -const logger = getLogger("server:nats:terminal"); +const logger = getLogger("project:conat:terminal:session"); + +// truncated excessive INPUT is CRITICAL to avoid deadlocking the terminal +// and completely crashing the project in case a user pastes in, e.g., +// a few hundred K, like this gist: https://gist.github.com/cheald/2905882 +// to a node session. Note VS code also crashes. +const MAX_INPUT_SIZE = 10000; +const INPUT_CHUNK_SIZE = 50; const EXIT_MESSAGE = "\r\n\r\n[Process completed - press any key]\r\n\r\n"; -// The printf at the end clears the line so the user doesn't see it. This took -// way too long to figure out how to do. See -// https://stackoverflow.com/questions/5861428/bash-script-erase-previous-line -// const COMPUTE_SERVER_PROMPT_MESSAGE = -// `PS1="(\\h) \\w$ "; history -d $(history 1); printf '\\e[A\\e[K'\n`; +const SOFT_RESET = + "tput rmcup; printf '\e[?1000l\e[?1002l\e[?1003l\e[?1006l\e[?1l'; clear -x; sleep 0.1; clear -x"; + +const COMPUTE_SERVER_INIT = `PS1="(\\h) \\w$ "; ${SOFT_RESET}; history -d $(history 1);\n`; -const COMPUTE_SERVER_PROMPT_MESSAGE = - 'PS1="(\\h) \\w$ ";reset;history -d $(history 1)\n'; +const PROJECT_INIT = `${SOFT_RESET}; history -d $(history 1);\n`; const DEFAULT_COMMAND = "/bin/bash"; const INFINITY = 999999; const HISTORY_LIMIT_BYTES = parseInt( - process.env.COCALC_TERMINAL_HISTORY_LIMIT_BYTES ?? "20000", + process.env.COCALC_TERMINAL_HISTORY_LIMIT_BYTES ?? "2000000", ); // Limits that result in dropping messages -- this makes sense for a terminal (unlike a file you're editing). -// Limit number of MB/s in data: +// Limit number of bytes per second in data: const MAX_BYTES_PER_SECOND = parseInt( - process.env.COCALC_TERMINAL_MAX_BYTES_PER_SECOND ?? "500000", + process.env.COCALC_TERMINAL_MAX_BYTES_PER_SECOND ?? "1000000", ); -// Limit number of messages per second. +// Hard limit at stream level the number of messages per second. +// However, the code in this file must already limit +// writing output less than this to avoid the stream ever +// having to discard writes. This is basically the "frame rate" +// we are supporting for users. const MAX_MSGS_PER_SECOND = parseInt( - process.env.COCALC_TERMINAL_MAX_MSGS_PER_SECOND ?? "250", -); - -const sessions: { [path: string]: Session } = {}; - -export async function createTerminalService(path: string) { - let options: any = undefined; - const getSession = async (noCreate?: boolean) => { - const cur = sessions[path]; - if (cur == null) { - if (noCreate) { - throw Error("no terminal session"); - } - await createTerminal({ ...options, path }); - const session = sessions[path]; - if (session == null) { - throw Error( - `BUG: failed to create terminal session - ${path} (this should not happen)`, - ); - } - return session; - } - return cur; - }; - const impl = { - create: async (opts: { - env?: { [key: string]: string }; - command?: string; - args?: string[]; - cwd?: string; - ephemeral?: boolean; - }): Promise<{ success: "ok"; note?: string; ephemeral?: boolean }> => { - // save options to reuse. - options = opts; - const note = await createTerminal({ ...opts, path }); - // passing back ephemeral is for backward compat, since old versions don't - // know about this, so they always pass back false, and the frontend can - // then use false. TODO: remove this later. - return { success: "ok", note, ephemeral: opts.ephemeral }; - }, - - write: async (data: string): Promise => { - if (typeof data != "string") { - throw Error(`data must be a string -- ${JSON.stringify(data)}`); - } - const session = await getSession(); - await session.write(data); - }, - - restart: async () => { - const session = await getSession(); - await session.restart(); - }, - - cwd: async () => { - const session = await getSession(); - return await session.getCwd(); - }, - - kill: async () => { - try { - const session = await getSession(true); - await session.close(); - } catch { - return; - } - }, - - size: async (opts: { - rows: number; - cols: number; - browser_id: string; - kick?: boolean; - }) => { - const session = await getSession(); - session.setSize(opts); - }, - - close: async (browser_id: string) => { - sessions[path]?.browserLeaving(browser_id); - }, - }; - - const server = await createTerminalServer({ path, project_id, impl }); - server.on("close", () => { - sessions[path]?.close(); - delete sessions[path]; - }); - return server; -} - -function closeTerminal(path: string) { - const cur = sessions[path]; - if (cur != null) { - cur.close(); - delete sessions[path]; - } -} - -export const createTerminal = reuseInFlight( - async (params) => { - if (params == null) { - throw Error("params must be specified"); - } - const { path, ...options } = params; - if (!path) { - throw Error("path must be specified"); - } - let note = ""; - const cur = sessions[path]; - if (cur != null) { - if (!isEqual(cur.options, options) || cur.state == "closed") { - // clean up -- we will make new one below - closeTerminal(path); - note += "Closed existing session. "; - } else { - // already have a working session with correct options - note += "Already have working session with same options. "; - return note; - } - } - note += "Creating new session."; - let session = new Session({ path, options }); - await session.init(); - if (session.state == "closed") { - // closed during init -- unlikely but possible; try one more time - session = new Session({ path, options }); - await session.init(); - if (session.state == "closed") { - throw Error(`unable to create terminal session for ${path}`); - } - } else { - sessions[path] = session; - return note; - } - }, - { - createKey: (args) => { - return args[0]?.path ?? ""; - }, - }, + process.env.COCALC_TERMINAL_MAX_MSGS_PER_SECOND ?? "24", ); type State = "running" | "off" | "closed"; -class Session { +export class Session { public state: State = "off"; - public options; - private path: string; + public options: CreateTerminalOptions; + private termPath: string; private pty?; private size?: { rows: number; cols: number }; private browserApi: ReturnType; @@ -207,13 +72,14 @@ class Session { private clientSizes: { [browser_id: string]: { rows: number; cols: number; time: number }; } = {}; + public pid: number; - constructor({ path, options }) { - logger.debug("create session ", { path, options }); - this.path = path; - this.browserApi = createBrowserClient({ project_id, path }); + constructor({ termPath, options }) { + logger.debug("create session ", { termPath, options }); + this.termPath = termPath; + this.browserApi = createBrowserClient({ project_id, termPath }); this.options = options; - this.streamName = `terminal-${path}`; + this.streamName = `terminal-${termPath}`; } write = async (data) => { @@ -223,7 +89,26 @@ class Session { // which you don't want to send except to start it. return; } - this.pty?.write(data); + let reject; + if (data.length > MAX_INPUT_SIZE) { + data = data.slice(0, MAX_INPUT_SIZE); + reject = true; + } else { + reject = false; + } + for ( + let i = 0; + i < data.length && this.pty != null; + i += INPUT_CHUNK_SIZE + ) { + const chunk = data.slice(i, i + INPUT_CHUNK_SIZE); + this.pty.write(chunk); + logger.debug("wrote data to pty", chunk.length); + await delay(1000 / MAX_MSGS_PER_SECOND); + } + if (reject) { + this.stream?.publish(`\r\n[excessive input discarded]\r\n\r\n`); + } }; restart = async () => { @@ -242,9 +127,6 @@ class Session { delete this.pty; delete this.stream; this.state = "closed"; - if (sessions[this.path] === this) { - delete sessions[this.path]; - } this.clientSizes = {}; }; @@ -273,47 +155,40 @@ class Session { this.stream = await dstream({ name: this.streamName, ephemeral: this.options.ephemeral, - // server side is THE leader. - leader: true, - limits: { + config: { max_bytes: HISTORY_LIMIT_BYTES, max_bytes_per_second: MAX_BYTES_PER_SECOND, max_msgs_per_second: MAX_MSGS_PER_SECOND, }, }); - this.stream.on("reject", ({ err }) => { - if (err.limit == "max_bytes_per_second") { - // instead, send something small - this.throttledEllipses(); - } else if (err.limit == "max_msgs_per_second") { - // only sometimes send [...], because channel is already full and it - // doesn't help to double the messages! - this.throttledEllipses(); - } + this.stream.publish("\r\n".repeat((this.size?.rows ?? 40) + 40)); + this.stream.on("reject", () => { + this.throttledEllipses(); }); }; private throttledEllipses = throttle( () => { - this.stream?.publish( - `\r\n[...excessive output discarded above...]\r\n\r\n`, - ); + this.stream?.publish(`\r\n[excessive output discarded]\r\n\r\n`); }, 1000, { leading: true, trailing: true }, ); init = async () => { - const { head, tail } = path_split(this.path); + const { head, tail } = path_split(this.termPath); + const HISTFILE = historyFile(this.options.path); const env = { - COCALC_TERMINAL_FILENAME: tail, - ...envForSpawn(), + PROMPT_COMMAND: "history -a", + ...(HISTFILE ? { HISTFILE } : undefined), ...this.options.env, + ...envForSpawn(), + COCALC_TERMINAL_FILENAME: tail, TMUX: undefined, // ensure not set }; const command = this.options.command ?? DEFAULT_COMMAND; const args = this.options.args ?? []; - const initFilename: string = console_init_filename(this.path); + const initFilename: string = console_init_filename(this.termPath); if (await exists(initFilename)) { args.push("--init-file"); args.push(path_split(initFilename).tail); @@ -328,23 +203,37 @@ class Session { env, rows: this.size?.rows, cols: this.size?.cols, + handleFlowControl: true, }); - if (compute_server_id && command == "/bin/bash") { - // set the prompt to show the remote hostname explicitly, - // then clear the screen. - this.pty.write(COMPUTE_SERVER_PROMPT_MESSAGE); + this.pid = this.pty.pid; + if (command.endsWith("bash")) { + if (compute_server_id) { + // set the prompt to show the remote hostname explicitly, + // then clear the screen. + this.pty.write(COMPUTE_SERVER_INIT); + } else { + this.pty.write(PROJECT_INIT); + } } this.state = "running"; logger.debug("creating stream"); await this.createStream(); + logger.debug("created the stream"); if ((this.state as State) == "closed") { return; } logger.debug("connect stream to pty"); - this.pty.onData((data: string) => { + + // use slighlty less than MAX_MSGS_PER_SECOND to avoid reject + // due to being *slightly* off. + const throttle = new Throttle(1000 / (MAX_MSGS_PER_SECOND - 3)); + throttle.on("data", (data: string) => { + logger.debug("got data out of pty"); this.handleBackendMessages(data); this.stream?.publish(data); }); + this.pty.onData(throttle.write); + this.pty.onExit(() => { this.stream?.publish(EXIT_MESSAGE); this.state = "off"; @@ -384,7 +273,7 @@ class Session { return; } const { rows, cols } = size; - logger.debug("resize", "new size", rows, cols); + // logger.debug("resize", "new size", rows, cols); try { this.setSizePty({ rows, cols }); // tell browsers about our new size @@ -521,3 +410,18 @@ function getCWD(pathHead, cwd?): string { } return pathHead; } + +function historyFile(path: string): string | undefined { + if (path.startsWith("/")) { + // only set histFile for paths in the home directory i.e., + // relative to HOME. Absolute paths -- we just leave it alone. + // E.g., the miniterminal uses /tmp/... for its path. + return undefined; + } + const { head, tail } = path_split(path); + return join( + process.env.HOME ?? "", + head, + tail.endsWith(".term") ? tail : ".bash_history", + ); +} diff --git a/src/packages/project/data.ts b/src/packages/project/data.ts index 594f5d3bc4..b7b5ac0c44 100644 --- a/src/packages/project/data.ts +++ b/src/packages/project/data.ts @@ -22,9 +22,15 @@ export const sessionIDFile = join(data, "session-id.txt"); export const rootSymlink = join(data, "root"); export const SSH_LOG = join(data, "sshd.log"); export const SSH_ERR = join(data, "sshd.err"); -export const secretToken = - process.env.COCALC_SECRET_TOKEN ?? join(data, "secret_token"); export const compute_server_id = parseInt(process.env.COMPUTE_SERVER_ID ?? "0"); +export const sageServerPaths = { + log: join(data, "sage_server", "sage_server.log"), + pid: join(data, "sage_server", "sage_server.pid"), + port: join(data, "sage_server", "sage_server.port"), +} as const; + +// secret token must be after compute_server_id is set, since it uses it. +export { secretToken } from "./secret-token"; // note that the "username" need not be the output of `whoami`, e.g., // when using a cc-in-cc dev project where users are "virtual". diff --git a/src/packages/project/http-api/get-syncdoc-history.ts b/src/packages/project/http-api/get-syncdoc-history.ts deleted file mode 100644 index 6f767e771a..0000000000 --- a/src/packages/project/http-api/get-syncdoc-history.ts +++ /dev/null @@ -1,35 +0,0 @@ -/* - -EXAMPLE: - -~$ curl -u `cat .smc/secret_token`: -d path=a.md http://127.0.0.1:`cat .smc/api-server.port`/api/v1/get-syncdoc-history -d patches | python -m json.tool - -If you get an error about no hubs connected, then edit a file in the project -in a browser to cause a connection to happen. Also, a.md need to be a file that -you have edited. -*/ - -import { client_db } from "@cocalc/util/db-schema"; -import { client } from "./server"; -import { syncdbPath } from "@cocalc/util/jupyter/names"; - -export default async function getSyncdocHistory({ - path, - patches, -}): Promise { - const dbg = client.dbg("get-syncdoc-history"); - dbg(`path="${path}"`); - if (typeof path != "string") { - throw Error("provide the path as a string"); - } - - // transform jupyter path -- TODO: this should - // be more centralized... since this is brittle. - if (path.endsWith(".ipynb")) { - path = syncdbPath(path); - } - - // compute the string_id - const string_id = client_db.sha1(client.project_id, path); - return await client.get_syncdoc_history(string_id, !!patches); -} diff --git a/src/packages/project/http-api/server.ts b/src/packages/project/http-api/server.ts index e60892374b..1540429e55 100644 --- a/src/packages/project/http-api/server.ts +++ b/src/packages/project/http-api/server.ts @@ -22,11 +22,9 @@ import RateLimit from "express-rate-limit"; import { writeFile } from "node:fs"; import { getOptions } from "@cocalc/project/init-program"; import { getClient } from "@cocalc/project/client"; -import { apiServerPortFile } from "@cocalc/project/data"; -import { getSecretToken } from "@cocalc/project/servers/secret-token"; +import { apiServerPortFile, secretToken } from "@cocalc/project/data"; import { once } from "@cocalc/util/async-utils"; import { split } from "@cocalc/util/misc"; -import getSyncdocHistory from "./get-syncdoc-history"; import readTextFile from "./read-text-file"; import writeTextFile from "./write-text-file"; @@ -54,7 +52,9 @@ export default async function init(): Promise { dbg(`writing port to file "${apiServerPortFile}"`); await callback(writeFile, apiServerPortFile, `${port}`); - dbg(`express server successfully listening at http://${options.hostname}:${port}`); + dbg( + `express server successfully listening at http://${options.hostname}:${port}`, + ); } function configure(server: express.Application, dbg: Function): void { @@ -109,9 +109,6 @@ function handleAuth(req): void { throw Error(`unknown authorization type '${type}'`); } - // could throw if not initialized yet -- done in ./init.ts via initSecretToken() - const secretToken = getSecretToken(); - // now check auth if (secretToken != providedToken) { throw Error(`incorrect secret token "${secretToken}", "${providedToken}"`); @@ -121,8 +118,6 @@ function handleAuth(req): void { async function handleEndpoint(req): Promise { const endpoint: string = req.path.slice(req.path.lastIndexOf("/") + 1); switch (endpoint) { - case "get-syncdoc-history": - return await getSyncdocHistory(getParams(req, ["path", "patches"])); case "write-text-file": return await writeTextFile(getParams(req, ["path", "content"])); case "read-text-file": diff --git a/src/packages/project/http-api/testing.js b/src/packages/project/http-api/testing.js deleted file mode 100644 index 1ea7c8fb89..0000000000 --- a/src/packages/project/http-api/testing.js +++ /dev/null @@ -1,37 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -require("ts-node").register({ - project: __dirname + "/../tsconfig.json", - cacheDirectory: "/tmp", -}); - -const client = { - secret_token: "secret", - project_id: "e11c1abe-52a0-4959-ac1a-391e14088bf5", - async get_syncdoc_history(string_id, patches) { - return [{ string_id, this_is_fake: true }]; - }, - dbg(name) { - return (...args) => { - console.log(name, ...args); - }; - }, -}; - -async function start() { - try { - await require("./server.ts").start_server({ - port: 8080, - port_path: "/tmp/port", - client, - }); - } catch (err) { - console.log(`EXCEPTION -- ${err}`); - console.trace(); - } -} - -start(); diff --git a/src/packages/project/jupyter/test/README.md b/src/packages/project/jupyter/test/README.md deleted file mode 100644 index 4ffb52eda2..0000000000 --- a/src/packages/project/jupyter/test/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Jupyter CoCalc Kernel Testing - -WARNING: This specific test directory has atrophied and is in no way supported right now. diff --git a/src/packages/project/jupyter/test/basic.ts b/src/packages/project/jupyter/test/basic.ts deleted file mode 100644 index f9bd72a6e8..0000000000 --- a/src/packages/project/jupyter/test/basic.ts +++ /dev/null @@ -1,54 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -import expect from "expect"; - -import { kernel, exec, JupyterKernel } from "./common"; - -describe("compute 2+3 using python2", function () { - this.timeout(10000); - let k: JupyterKernel; - - it("creates a python2 kernel", function () { - k = kernel("test-python2"); - }); - - it("spawn", async function () { - await k.spawn(); - expect(k.get_state()).toBe("running"); - }); - - it("evaluate 2+3", async function () { - expect(await exec(k, "2+3")).toEqual('{"text/plain":"5"}'); - }); - - it("closes the kernel", function () { - k.close(); - expect(k.get_state()).toBe("closed"); - }); -}); - -describe("compute 2/3 using python3", function () { - this.timeout(10000); - let k: JupyterKernel; - - it("creates a python3 kernel", function () { - k = kernel("test-python3"); - }); - - it("spawn", async function () { - await k.spawn(); - expect(k.get_state()).toBe("running"); - }); - - it("evaluate 2/3", async function () { - expect(await exec(k, "2/3")).toEqual('{"text/plain":"0.6666666666666666"}'); - }); - - it("closes the kernel", function () { - k.close(); - expect(k.get_state()).toBe("closed"); - }); -}); diff --git a/src/packages/project/jupyter/test/common.ts b/src/packages/project/jupyter/test/common.ts deleted file mode 100644 index 4e1463ba05..0000000000 --- a/src/packages/project/jupyter/test/common.ts +++ /dev/null @@ -1,62 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2023 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -// Use clean in-memory blob store for tests. -process.env.JUPYTER_BLOBS_DB_FILE = "memory"; - -import { kernel as jupyter_kernel } from "../jupyter"; - -//import type { JupyterKernelInterface } from "@cocalc/frontend/jupyter/project-interface"; -type JupyterKernelInterface = any; -export type JupyterKernel = JupyterKernelInterface; - -import json from "json-stable-stringify"; - -const DEBUG = !!process.env["DEBUG"]; -if (DEBUG) { - console.log("DEBUG =", DEBUG); -} - -// We use custom kernels for testing, since faster to start. -// For example, we don't use matplotlib inline for testing (much) and -// using it greatly slows down startup. -export function custom_kernel_path() { - process.env.JUPYTER_PATH = `${__dirname}/jupyter`; -} -custom_kernel_path(); - -export function default_kernel_path() { - process.env.JUPYTER_PATH = "/ext/jupyter"; -} - -export function kernel(name: string, path?: string): JupyterKernelInterface { - if (path == null) { - path = ""; - } - return jupyter_kernel({ name, verbose: DEBUG, path }); -} - -export async function exec(k: JupyterKernel, code: string): Promise { - return output(await k.execute_code_now({ code: code })); -} - -// String summary of key aspect of output, which is useful for testing. -export function output(v: any[]): string { - let s = ""; - let x: any; - for (x of v) { - if (x.content == null) continue; - if (x.content.data != null) { - return json(x.content.data); - } - if (x.content.text != null) { - s += x.content.text; - } - if (x.content.ename != null) { - return json(x.content); - } - } - return s; -} diff --git a/src/packages/project/jupyter/test/completion.ts b/src/packages/project/jupyter/test/completion.ts deleted file mode 100644 index e0e1219945..0000000000 --- a/src/packages/project/jupyter/test/completion.ts +++ /dev/null @@ -1,117 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -/* -Test completion API -*/ - -import {} from "mocha"; -import * as expect from "expect"; -import * as common from "./common"; - -// global kernel being tested at any point. -let kernel: common.JupyterKernel; - -// This checks that on input the given obj={code:?, cursor_pos:?} -// the resulting matches *contains* matches -function check(obj: { code: string; cursor_pos: any }, matches?: string[]) { - it(`checks that ${JSON.stringify(obj)} includes ${ - matches ? JSON.stringify(matches) : "nothing" - }`, async function () { - const resp = await kernel.complete({ - code: obj.code, - cursor_pos: obj.cursor_pos != null ? obj.cursor_pos : obj.code.length, - }); - if (matches === undefined) { - expect(resp.matches.length).toBe(0); - } else { - for (const m of matches) { - expect(resp.matches).toContain(m); - } - } - }); -} - -describe("complete some things using python2 kernel -- ", function () { - this.timeout(10000); - - it("creates a python2 kernel", function () { - kernel = common.kernel("test-python2"); - }); - - it("completes 'imp'", async function () { - const resp = await kernel.complete({ - code: "imp", - cursor_pos: 2, - }); - expect(resp).toEqual({ - matches: ["import"], - status: "ok", - cursor_start: 0, - cursor_end: 2, - }); - }); - - check({ code: "imp", cursor_pos: 3 }, ["import"]); - check({ code: "in", cursor_pos: 2 }, ["in", "input", "int", "intern"]); - check({ code: "in", cursor_pos: 1 }, [ - "id", - "if", - "import", - "in", - "input", - "int", - "intern", - "is", - "isinstance", - "issubclass", - "iter", - ]); - - check({ code: "alsdfl", cursor_pos: 5 }); - - it("creates a new identifier", async function () { - await common.exec(kernel, 'alsdfl = {"foo":"bar"}'); - }); - - check({ code: "alsdfl", cursor_pos: 6 }, ["alsdfl"]); - - check({ code: "alsdfl._", cursor_pos: 8 }, [ - "alsdfl.__class__", - "alsdfl.__cmp__", - ]); - - it("closes the kernel", () => kernel.close()); -}); - -describe("complete some things using sage kernel -- ", function () { - this.timeout(30000); // sage can be very slow to start. - - it("creates a sage kernel", function () { - kernel = common.kernel("test-sagemath"); - }); - - check({ code: "Ell", cursor_pos: 3 }, [ - "Ellipsis", - "EllipticCurve", - "EllipticCurve_from_c4c6", - "EllipticCurve_from_cubic", - "EllipticCurve_from_j", - "EllipticCurve_from_plane_curve", - "EllipticCurveIsogeny", - "EllipticCurves_with_good_reduction_outside_S", - ]); - - check({ code: "e.", cursor_pos: 2 }, [ - "e.abs", - "e.add", - "e.add_to_both_sides", - "e.additive_order", - "e.arccos", - ]); - check({ code: "e.fac", cursor_pos: 5 }, ["e.factor"]); - - it("closes the kernel", () => kernel.close()); -}); diff --git a/src/packages/project/jupyter/test/jupyter/kernels/sagemath/kernel.json b/src/packages/project/jupyter/test/jupyter/kernels/sagemath/kernel.json deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/packages/project/jupyter/test/jupyter/kernels/test-ir/kernel.json b/src/packages/project/jupyter/test/jupyter/kernels/test-ir/kernel.json deleted file mode 100755 index 0db2d3cbc8..0000000000 --- a/src/packages/project/jupyter/test/jupyter/kernels/test-ir/kernel.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "language": "r", - "argv": ["R", "-e", "IRkernel::main()", "--args", "{connection_file}"], - "display_name": "R (R-Project)" -} diff --git a/src/packages/project/jupyter/test/jupyter/kernels/test-python2/kernel.json b/src/packages/project/jupyter/test/jupyter/kernels/test-python2/kernel.json deleted file mode 100755 index a2423d6384..0000000000 --- a/src/packages/project/jupyter/test/jupyter/kernels/test-python2/kernel.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "display_name": "Python 2 (Ubuntu Linux)", - "argv": [ - "/ext/bin/python2-ubuntu", - "-m", - "ipykernel", - "-f", - "{connection_file}" - ], - "language": "python" -} diff --git a/src/packages/project/jupyter/test/jupyter/kernels/test-python3/kernel.json b/src/packages/project/jupyter/test/jupyter/kernels/test-python3/kernel.json deleted file mode 100755 index 76dde844ac..0000000000 --- a/src/packages/project/jupyter/test/jupyter/kernels/test-python3/kernel.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "display_name": "Python 2 (Ubuntu Linux)", - "argv": ["/usr/bin/python3", "-m", "ipykernel", "-f", "{connection_file}"], - "language": "python" -} diff --git a/src/packages/project/jupyter/test/jupyter/kernels/test-sagemath/kernel.json b/src/packages/project/jupyter/test/jupyter/kernels/test-sagemath/kernel.json deleted file mode 100644 index b8ebd87d01..0000000000 --- a/src/packages/project/jupyter/test/jupyter/kernels/test-sagemath/kernel.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "display_name": "SageMath (stable)", - "argv": [ - "sage", - "--python", - "-m", - "sage.repl.ipython_kernel", - "-f", - "{connection_file}" - ] -} diff --git a/src/packages/project/jupyter/test/kernel.ts b/src/packages/project/jupyter/test/kernel.ts deleted file mode 100644 index f4d1c0582d..0000000000 --- a/src/packages/project/jupyter/test/kernel.ts +++ /dev/null @@ -1,154 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -import {} from "mocha"; -import * as expect from "expect"; -import * as common from "./common"; -import { endswith } from "@cocalc/util/misc"; - -describe("compute 2+7 using the python2 kernel -- ", function () { - this.timeout(5000); - const kernel: common.JupyterKernel = common.kernel("test-python2"); - - it("evaluate 2+7", async function () { - expect(await common.exec(kernel, "2+7")).toBe('{"text/plain":"9"}'); - }); - - it("closes the kernel", function () { - kernel.close(); - }); - - it("verifies that executing code after closing the kernel gives an appropriate error", async function () { - try { - await kernel.execute_code_now({ code: "2+2" }); - } catch (err) { - expect(err.toString()).toBe("Error: closed"); - } - }); -}); - -describe("compute 2/3 using a python3 kernel -- ", function () { - this.timeout(15000); - const kernel: common.JupyterKernel = common.kernel("test-python3"); - - it("evaluate 2/3", async function () { - expect(await common.exec(kernel, "2/3")).toBe( - '{"text/plain":"0.6666666666666666"}' - ); - }); - - return it("closes the kernel", function () { - kernel.close(); - }); -}); - -describe("it tries to start a kernel that does not exist -- ", function () { - let kernel: common.JupyterKernel; - - it("creates a foobar kernel", function () { - kernel = common.kernel("foobar"); - return expect(kernel.get_state()).toBe("off"); - }); - - it("then tries to use it, which will fail", async function () { - try { - await kernel.execute_code_now({ code: "2+2" }); - } catch (err) { - expect(err.toString()).toBe("Error: No spec available for foobar"); - } - }); -}); - -describe("calling the spawn method -- ", function () { - const kernel = common.kernel("test-python2"); - this.timeout(5000); - - it("observes that the state switches to running", function (done) { - kernel.on("state", function (state) { - if (state !== "running") { - return; - } - done(); - }); - kernel.spawn(); - }); - - it("observes that the state switches to closed", function (done) { - kernel.on("state", function (state) { - if (state !== "closed") { - return; - } - done(); - }); - kernel.close(); - }); -}); - -describe("send signals to a kernel -- ", function () { - const kernel = common.kernel("test-python2"); - this.timeout(5000); - - it("ensure kernel is running", async function () { - await kernel.spawn(); - }); - - it("start a long sleep running... and interrupt it", async function () { - // send an interrupt signal to stop the sleep below: - return setTimeout(() => kernel.signal("SIGINT"), 250); - expect(await common.exec(kernel, "import time; time.sleep(1000)")).toBe( - "foo" - ); - }); - - it("send a kill signal", function (done) { - kernel.on("state", function (state) { - expect(state).toBe("closed"); - done(); - }); - kernel.signal("SIGKILL"); - }); -}); - -describe("start a kernel in a different directory -- ", function () { - let kernel: common.JupyterKernel; - this.timeout(5000); - - it("creates a python2 kernel in current dir", async function () { - kernel = common.kernel("test-python2"); - expect( - endswith( - await common.exec(kernel, 'import os; print(os.path.abspath("."))'), - "project/jupyter\n" - ) - ).toBe(true); - kernel.close(); - }); - - it("creates a python2 kernel with path test/a.ipynb2", async function () { - kernel = common.kernel("test-python2", "test/a.ipynb2"); - expect( - endswith( - await common.exec(kernel, 'import os; print(os.path.abspath("."))'), - "project/jupyter/test\n" - ) - ).toBe(true); - kernel.close(); - }); -}); - -describe("use the key:value store -- ", function () { - const kernel = common.kernel("test-python2"); - this.timeout(5000); - - it("tests setting the store", function () { - kernel.store.set({ a: 5, b: 7 }, { the: "value" }); - expect(kernel.store.get({ b: 7, a: 5 })).toEqual({ the: "value" }); - }); - - it("tests deleting from the store", function () { - kernel.store.delete({ a: 5, b: 7 }); - expect(kernel.store.get({ b: 7, a: 5 })).toBe(undefined); - }); -}); diff --git a/src/packages/project/jupyter/test/mocha.opts b/src/packages/project/jupyter/test/mocha.opts deleted file mode 100644 index ff5ad97489..0000000000 --- a/src/packages/project/jupyter/test/mocha.opts +++ /dev/null @@ -1,3 +0,0 @@ ---require ts-node/register ---reporter spec ---recursive diff --git a/src/packages/project/jupyter/test/payload.ts b/src/packages/project/jupyter/test/payload.ts deleted file mode 100644 index 115695726f..0000000000 --- a/src/packages/project/jupyter/test/payload.ts +++ /dev/null @@ -1,51 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -/* -Test payload shell message. -*/ - -import {} from "mocha"; -import * as expect from "expect"; -import * as common from "./common"; -import { startswith, getIn } from "@cocalc/util/misc"; - -describe("create python2 kernel and do evals with and without payloads -- ", async function () { - this.timeout(5000); - - const kernel: common.JupyterKernel = common.kernel("test-python2"); - - it("does an eval with no payload", async function () { - const result: any[] = await kernel.execute_code_now({ - code: "2+3", - }); - for (const x of result) { - if (getIn(x, ["content", "payload"], []).length > 0) { - throw Error("there should not be any payloads"); - } - } - }); - - it("does an eval with a payload (requires internet)", async function () { - const result: any[] = await kernel.execute_code_now({ - code: - "%load https://matplotlib.org/mpl_examples/showcase/integral_demo.py", - }); - for (const x of result) { - let v; - if ((v = getIn(x, ["content", "payload"], [])).length > 0) { - const s = - '# %load https://matplotlib.org/mpl_examples/showcase/integral_demo.py\n"""\nPlot demonstrating'; - expect(v.length).toBe(1); - expect(startswith(v[0].text, s)).toBe(true); - return; - } - } - }); - - return it("closes the kernel", function () { - kernel.close(); - }); -}); diff --git a/src/packages/project/jupyter/test/stdin.ts b/src/packages/project/jupyter/test/stdin.ts deleted file mode 100644 index 034fb176e0..0000000000 --- a/src/packages/project/jupyter/test/stdin.ts +++ /dev/null @@ -1,116 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -/* -Test backend part of interactive input. -*/ - -import {} from "mocha"; -import * as expect from "expect"; -import * as common from "./common"; - -function stdin_func(expected_prompt, expected_password, value) { - async function stdin(prompt: string, password: boolean): Promise { - if (prompt != expected_prompt) { - return `'bad prompt=${prompt}'`; - } - if (password != expected_password) { - return `'password=${password} should not be set'`; - } - return JSON.stringify(value); - } - return stdin; -} - -describe("get input using the python2 kernel -- ", function () { - this.timeout(10000); - let kernel: common.JupyterKernel; - - it("creates a python2 kernel", function () { - kernel = common.kernel("test-python2"); - }); - - it("reading input - no prompt", async function () { - const out = await kernel.execute_code_now({ - code: "print(input())", - stdin: stdin_func("", false, "cocalc"), - }); - expect(common.output(out)).toEqual("cocalc\n"); - }); - - it("reading input - different return", async function () { - const out = await kernel.execute_code_now({ - code: "print(input())", - stdin: stdin_func("", false, "sage"), - }); - expect(common.output(out)).toEqual("sage\n"); - }); - - it("reading raw_input - no prompt", async function () { - const out = await kernel.execute_code_now({ - code: "print(raw_input())", - stdin: stdin_func("", false, "cocalc"), - }); - expect(common.output(out)).toEqual('"cocalc"\n'); - }); - - it("reading input - prompt", async function () { - const out = await kernel.execute_code_now({ - code: 'print(input("prompt"))', - stdin: stdin_func("prompt", false, "cocalc"), - }); - expect(common.output(out)).toEqual("cocalc\n"); - }); - - it("reading raw_input - prompt", async function () { - const out = await kernel.execute_code_now({ - code: 'print(raw_input("prompt"))', - stdin: stdin_func("prompt", false, "cocalc"), - }); - expect(common.output(out)).toEqual('"cocalc"\n'); - }); - - it("reading a password", async function () { - const out = await kernel.execute_code_now({ - code: 'import getpass; print(getpass.getpass("password?"))', - stdin: stdin_func("password?", true, "cocalc"), - }); - expect(common.output(out)).toEqual('"cocalc"\n'); - }); - - return it("closes the kernel", function () { - kernel.close(); - }); -}); - -/* -describe("get input using the python3 kernel -- ", function() { - this.timeout(20000); - - it("do it", async function() { - const kernel = common.kernel("test-python3"); - const out = await kernel.execute_code_now({ - code: 'print(input("prompt"))', - stdin: stdin_func("prompt", false, "cocalc") - }); - expect(common.output(out)).toEqual("cocalc\n"); - }); -}); - -describe("get input using the ir kernel -- ", function() { - this.timeout(20000); - - it("do it", async function() { - const kernel = common.kernel("test-ir"); - const out = await kernel.execute_code_now({ - code: 'print(readline("prompt"))', - stdin: stdin_func("prompt", false, "cocalc") - }); - expect(common.output(out)).toEqual('[1] "cocalc"\n'); - kernel.close(); - }); -}); - -*/ diff --git a/src/packages/project/jupyter/test/supported-kernels.ts b/src/packages/project/jupyter/test/supported-kernels.ts deleted file mode 100644 index f40d986e36..0000000000 --- a/src/packages/project/jupyter/test/supported-kernels.ts +++ /dev/null @@ -1,212 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -/* -Do some tests with **all** of the kernels we officially support in CoCalc. - -Obviously, this test file should take a while to run, since it starts -up many, many kernels, several of which are old Sage versions. - -Also, sadly the tests are flakie... since some Jupyter kernels are flakie. -*/ - -import {} from "mocha"; -import * as expect from "expect"; -import * as common from "./common"; - -/* As of Aug, 2018: - -$ jupyter kernelspec list - - anaconda3 /ext/jupyter/kernels/anaconda3 - anaconda5 /ext/jupyter/kernels/anaconda5 - bash /ext/jupyter/kernels/bash - calysto_prolog /ext/jupyter/kernels/calysto_prolog - gap /ext/jupyter/kernels/gap - haskell /ext/jupyter/kernels/haskell - ir /ext/jupyter/kernels/ir - ir-sage /ext/jupyter/kernels/ir-sage - julia /ext/jupyter/kernels/julia - octave /ext/jupyter/kernels/octave - pari_jupyter /ext/jupyter/kernels/pari_jupyter - python2 /ext/jupyter/kernels/python2 - python2-ubuntu /ext/jupyter/kernels/python2-ubuntu - python3 /ext/jupyter/kernels/python3 - sage-8.1 /ext/jupyter/kernels/sage-8.1 - sage-8.2 /ext/jupyter/kernels/sage-8.2 - sage-develop /ext/jupyter/kernels/sage-develop - sagemath /ext/jupyter/kernels/sagemath - singular /ext/jupyter/kernels/singular - vpython /ext/jupyter/kernels/vpython -*/ - -// Set only to focus on only one of the kernels below. -const ONLY = ""; - -interface OneTest { - input: string; - output: string; -} - -interface TestKernel { - kernel: string; - tests: OneTest[]; - timeout?: number; // in ms -} - -const EXEC_TESTS: TestKernel[] = [ - { - kernel: "anaconda3", - tests: [ - { - input: "print(1/3,4/3)", - output: "0.3333333333333333 1.3333333333333333\n", - }, - ], - }, - { - kernel: "anaconda5", - tests: [ - { - input: "print(1/3,4/3)", - output: "0.3333333333333333 1.3333333333333333\n", - }, - ], - }, - { - kernel: "bash", - tests: [{ input: "echo 'foo bar'", output: "foo bar\n" }], - }, - { - kernel: "gap", - tests: [{ input: "1/3 + 4/3", output: "5/3" }], - }, - { - kernel: "haskell", - tests: [ - { input: "1/3 + 4/3", output: '{"text/plain":"1.6666666666666665"}' }, - ], - }, - { - kernel: "ir", - tests: [ - { - input: "1/3 + 4/3", - output: - '{"text/html":"1.66666666666667","text/latex":"1.66666666666667","text/markdown":"1.66666666666667","text/plain":"[1] 1.666667"}', - }, - ], - }, - { - kernel: "ir-sage", - tests: [ - { - input: "1/3 + 4/3", - output: - '{"text/html":"1.66666666666667","text/latex":"1.66666666666667","text/markdown":"1.66666666666667","text/plain":"[1] 1.666667"}', - }, - ], - }, - { - kernel: "julia", - tests: [ - { input: "1/3 + 4/3", output: '{"text/plain":"1.6666666666666665"}' }, - ], - }, - { - kernel: "octave", - tests: [{ input: "1/3 + 4/3", output: "ans = 1.6667\n" }], - }, - { - kernel: "pari_jupyter", - tests: [{ input: "1/3 + 4/3", output: '{"text/plain":"5/3"}' }], - timeout: 20000, - }, - { - kernel: "python2", - tests: [{ input: "print(1/3,4/3)", output: "(0, 1)\n" }], - }, - { - kernel: "python2-ubuntu", - tests: [{ input: "print(1/3,4/3)", output: "(0, 1)\n" }], - }, - { - kernel: "python3", - tests: [ - { - input: "print(1/3,4/3)", - output: "0.3333333333333333 1.3333333333333333\n", - }, - ], - }, - { - kernel: "sage-8.2", - tests: [{ input: "1/3 + 4/3", output: '{"text/plain":"5/3"}' }], - timeout: 60000, - }, - { - kernel: "sage-8.3", - tests: [{ input: "1/3 + 4/3", output: '{"text/plain":"5/3"}' }], - timeout: 60000, - }, - { - kernel: "sage-develop", - tests: [{ input: "1/3 + 4/3", output: '{"text/plain":"5/3"}' }], - timeout: 60000, - }, - { - kernel: "sagemath", - tests: [{ input: "1/3 + 4/3", output: '{"text/plain":"5/3"}' }], - timeout: 60000, - }, - { - /* Rant here: https://github.com/sagemathinc/cocalc/issues/3071 */ - kernel: "singular", - tests: [ - { - input: "2 + 3", - output: - '{"text/plain":"5\\n skipping text from `(` error at token `)`\\n"}', - }, - ], - timeout: 30000, - }, - { - kernel: "vpython", - tests: [ - { - input: "print(1/3,4/3)", - output: "0.3333333333333333 1.3333333333333333\n", - }, - ], - }, -]; - -for (const test of EXEC_TESTS) { - if (ONLY && ONLY != test.kernel) { - continue; - } - describe(`tests the "${test.kernel}" kernel -- `, function () { - before(common.default_kernel_path); - after(common.custom_kernel_path); - this.timeout(test.timeout ? test.timeout : 20000); - - let kernel: common.JupyterKernel; - - it(`creates the "${test.kernel}" kernel`, function () { - kernel = common.kernel(test.kernel); - }); - - for (const { input, output } of test.tests) { - it(`evaluates "${input}"`, async function () { - expect(await common.exec(kernel, input)).toBe(output); - }); - } - - it(`closes the ${test.kernel} kernel`, function () { - kernel.close(); - }); - }); -} diff --git a/src/packages/project/lean/global-packages.ts b/src/packages/project/lean/global-packages.ts deleted file mode 100644 index 26506a76be..0000000000 --- a/src/packages/project/lean/global-packages.ts +++ /dev/null @@ -1,31 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -/* -Initialize the ~/.lean directory if it doesn't already exist. - -Right now we "install", i.e., make a pointer to a global copy -of mathlib if there is one. - -We might install more later. - -NOTE: Rob Lewis suggested this and also tested installing other things -later (e.g., a new mathlib) and says it works: "Okay, I think you're -safe with this setup. Run leanpkg install /ext/... in/below ~. -If someone creates a Lean project anywhere in ~, the new leanpkg.path -seems to override the global install. This happens whether or not -mathlib is added to the project with leanpkg add. The global install -is unavailable in the project as soon as the project is created." - -See https://github.com/sagemathinc/cocalc/issues/4393. -*/ - -import { executeCode } from "@cocalc/backend/execute-code"; - -export async function init_global_packages(): Promise { - const command = `[ ! -d "${process.env.HOME}/.lean" ] && [ -d /ext/lean/lean/mathlib ] && leanpkg install /ext/lean/lean/mathlib`; - // err_on_exit = false because nonzero exit code whenever we don't run the install, which is fine. - await executeCode({ command, bash: true, err_on_exit: false }); -} diff --git a/src/packages/project/lean/lean.ts b/src/packages/project/lean/lean.ts deleted file mode 100644 index 4f7523f41e..0000000000 --- a/src/packages/project/lean/lean.ts +++ /dev/null @@ -1,316 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -import { callback, delay } from "awaiting"; -import * as lean_client from "lean-client-js-node"; -import { isEqual } from "lodash"; -import { EventEmitter } from "node:events"; - -import type { Client } from "@cocalc/project/client"; -import { once } from "@cocalc/util/async-utils"; -import { close } from "@cocalc/util/misc"; -import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; - -type SyncString = any; -type LeanServer = any; - -// do not try to sync with lean more frequently than this -// unless it is completing quickly. -const SYNC_INTERVAL: number = 6000; - -// What lean has told us about a given file. -type Message = any; -type Messages = Message[]; - -// What lean has told us about all files. -interface State { - tasks: any; - paths: { [key: string]: Messages }; -} - -export class Lean extends EventEmitter { - paths: { [key: string]: SyncString } = {}; - client: Client; - _server: LeanServer | undefined; - _state: State = { tasks: [], paths: {} }; - private running: { [key: string]: number } = {}; - dbg: Function; - - constructor(client: Client) { - super(); - this.server = reuseInFlight(this.server.bind(this)); - this.client = client; - this.dbg = this.client.dbg("LEAN SERVER"); - this.running = {}; - } - - close(): void { - this.kill(); - close(this); - } - - is_running(path: string): boolean { - return !!this.running[path] && now() - this.running[path] < SYNC_INTERVAL; - } - - // nothing actually async in here... yet. - private async server(): Promise { - if (this._server != undefined) { - if (this._server.alive()) { - return this._server; - } - // Kill cleans up any assumptions about stuff - // being sync'd. - this.kill(); - // New server will now be created... below. - } - this._server = new lean_client.Server( - new lean_client.ProcessTransport( - "lean", - process.env.HOME ? process.env.HOME : ".", // satisfy typescript. - ["-M 4096"], - ), - ); - this._server.error.on((err) => this.dbg("error:", err)); - this._server.allMessages.on((allMessages) => { - this.dbg("messages: ", allMessages); - - const new_messages = {}; - for (const x of allMessages.msgs) { - const path: string = x.file_name; - delete x.file_name; - if (new_messages[path] === undefined) { - new_messages[path] = [x]; - } else { - new_messages[path].push(x); - } - } - - for (const path in this._state.paths) { - this.dbg("messages for ", path, new_messages[path]); - if (new_messages[path] === undefined) { - new_messages[path] = []; - } - this.dbg( - "messages for ", - path, - new_messages[path], - this._state.paths[path], - ); - // length 0 is a special case needed when going from pos number of messages to none. - if ( - new_messages[path].length === 0 || - !isEqual(this._state.paths[path], new_messages[path]) - ) { - this.dbg("messages for ", path, "EMIT!"); - this.emit("messages", path, new_messages[path]); - this._state.paths[path] = new_messages[path]; - } - } - }); - - this._server.tasks.on((currentTasks) => { - const { tasks } = currentTasks; - this.dbg("tasks: ", tasks); - const running = {}; - for (const task of tasks) { - running[task.file_name] = true; - } - for (const path in running) { - const v: any[] = []; - for (const task of tasks) { - if (task.file_name === path) { - delete task.file_name; // no longer needed - v.push(task); - } - } - this.emit("tasks", path, v); - } - for (const path in this.running) { - if (!running[path]) { - this.dbg("server", path, " done; no longer running"); - this.running[path] = 0; - this.emit("tasks", path, []); - if (this.paths[path].changed) { - // file changed while lean was running -- so run lean again. - this.dbg( - "server", - path, - " changed while running, so running again", - ); - this.paths[path].on_change(); - } - } - } - }); - this._server.connect(); - return this._server; - } - - // Start learn server parsing and reporting info about the given file. - // It will get updated whenever the file change. - async register(path: string): Promise { - this.dbg("register", path); - if (this.paths[path] !== undefined) { - this.dbg("register", path, "already registered"); - return; - } - // get the syncstring and start updating based on content - let syncstring: any = undefined; - while (syncstring == null) { - // todo change to be event driven! - syncstring = this.client.syncdoc({ path }); - if (syncstring == null) { - await delay(1000); - } else if (syncstring.get_state() != "ready") { - await once(syncstring, "ready"); - } - } - const on_change = async () => { - this.dbg("sync", path); - if (syncstring._closed) { - this.dbg("sync", path, "closed"); - return; - } - if (this.is_running(path)) { - // already running, so do nothing - it will rerun again when done with current run. - this.dbg("sync", path, "already running"); - this.paths[path].changed = true; - return; - } - - const value: string = syncstring.to_str(); - if (this.paths[path].last_value === value) { - this.dbg("sync", path, "skipping sync since value did not change"); - return; - } - if (value.trim() === "") { - this.dbg( - "sync", - path, - "skipping sync document is empty (and LEAN behaves weird in this case)", - ); - this.emit("sync", path, syncstring.hash_of_live_version()); - return; - } - this.paths[path].last_value = value; - this._state.paths[path] = []; - this.running[path] = now(); - this.paths[path].changed = false; - this.dbg("sync", path, "causing server sync now"); - await (await this.server()).sync(path, value); - this.emit("sync", path, syncstring.hash_of_live_version()); - }; - this.paths[path] = { - syncstring, - on_change, - }; - syncstring.on("change", on_change); - if (!syncstring._closed) { - on_change(); - } - syncstring.on("closed", () => { - this.unregister(path); - }); - } - - // Stop updating given file on changes. - unregister(path: string): void { - this.dbg("unregister", path); - if (!this.paths[path]) { - // not watching it - return; - } - const x = this.paths[path]; - delete this.paths[path]; - delete this.running[path]; - x.syncstring.removeListener("change", x.on_change); - x.syncstring.close(); - } - - // Kill the lean server and unregister all paths. - kill(): void { - this.dbg("kill"); - if (this._server != undefined) { - for (const path in this.paths) { - this.unregister(path); - } - this._server.dispose(); - delete this._server; - } - } - - async restart(): Promise { - this.dbg("restart"); - if (this._server != undefined) { - for (const path in this.paths) { - this.unregister(path); - } - await this._server.restart(); - } - } - - async info( - path: string, - line: number, - column: number, - ): Promise { - this.dbg("info", path, line, column); - if (!this.paths[path]) { - this.register(path); - await callback((cb) => this.once(`sync-#{path}`, cb)); - } - return await (await this.server()).info(path, line, column); - } - - async complete( - path: string, - line: number, - column: number, - skipCompletions?: boolean, - ): Promise { - this.dbg("complete", path, line, column); - if (!this.paths[path]) { - this.register(path); - await callback((cb) => this.once(`sync-#{path}`, cb)); - } - const resp = await ( - await this.server() - ).complete(path, line, column, skipCompletions); - //this.dbg("complete response", path, line, column, resp); - return resp; - } - - async version(): Promise { - return (await this.server()).getVersion(); - } - - // Return state of parsing for everything that is currently registered. - state(): State { - return this._state; - } - - messages(path: string): any[] { - const x = this._state.paths[path]; - if (x !== undefined) { - return x; - } - return []; - } -} - -let singleton: Lean | undefined; - -// Return the singleton lean instance. The client is assumed to never change. -export function lean_server(client: Client): Lean { - if (singleton === undefined) { - singleton = new Lean(client); - } - return singleton; -} - -function now(): number { - return Date.now(); -} diff --git a/src/packages/project/lean/server.ts b/src/packages/project/lean/server.ts deleted file mode 100644 index 34d3f6dc46..0000000000 --- a/src/packages/project/lean/server.ts +++ /dev/null @@ -1,180 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -/* -LEAN server -*/ - -const lean_files = {}; - -import { lean_server, Lean } from "./lean"; -import { init_global_packages } from "./global-packages"; -import { isEqual } from "lodash"; - -let the_lean_server: Lean | undefined = undefined; - -function init_lean_server(client: any, logger: any): void { - the_lean_server = lean_server(client); - the_lean_server.on("tasks", function (path: string, tasks: object[]) { - logger.debug("lean_server:websocket:tasks -- ", path, tasks); - const lean_file = lean_files[`lean:${path}`]; - if (lean_file !== undefined && !isEqual(lean_file.tasks, tasks)) { - lean_file.tasks = tasks; - lean_file.channel.write({ tasks }); - } - }); - - the_lean_server.on("sync", function (path: string, hash: number) { - logger.debug("lean_server:websocket:sync -- ", path, hash); - const lean_file = lean_files[`lean:${path}`]; - if (lean_file !== undefined && !isEqual(lean_file.sync, hash)) { - const sync = { hash: hash, time: Date.now() }; - lean_file.sync = sync; - lean_file.channel.write({ sync }); - } - }); - - the_lean_server.on("messages", function (path: string, messages: object) { - logger.debug("lean_server:websocket:messages -- ", path, messages); - const lean_file = lean_files[`lean:${path}`]; - if (lean_file !== undefined && !isEqual(lean_file.messages, messages)) { - lean_file.messages = messages; - lean_file.channel.write({ messages }); - } - }); -} - -export async function lean_channel( - client: any, - primus: any, - logger: any, - path: string -): Promise { - if (the_lean_server === undefined) { - await init_global_packages(); - init_lean_server(client, logger); - if (the_lean_server === undefined) { - // just to satisfy typescript. - throw Error("lean server not defined"); - } - } - the_lean_server.register(path); - - // TODO: delete lean_files[name] under some condition. - const name = `lean:${path}`; - if (lean_files[name] !== undefined) { - return name; - } - const channel = primus.channel(name); - lean_files[name] = { - channel, - messages: [], - }; - - channel.on("connection", function (spark: any): void { - // make sure lean server cares: - if (the_lean_server === undefined) { - // just to satisfy typescript. - throw Error("lean server not defined"); - } - the_lean_server.register(path); - - const lean_file = lean_files[name]; - if (lean_file === undefined) { - return; - } - // Now handle the connection - logger.debug( - "lean channel", - `new connection from ${spark.address.ip} -- ${spark.id}` - ); - spark.write({ - messages: lean_file.messages, - sync: lean_file.sync, - tasks: lean_file.tasks, - }); - spark.on("end", function () {}); - }); - - return name; -} - -function assert_type(name: string, x: any, type: string): void { - if (typeof x != type) { - throw Error(`${name} must have type ${type}`); - } -} - -export async function lean( - client: any, - _: any, - logger: any, - opts: any -): Promise { - if (the_lean_server === undefined) { - init_lean_server(client, logger); - if (the_lean_server === undefined) { - // just to satisfy typescript. - throw Error("lean server not defined"); - } - } - if (opts == null || typeof opts.cmd != "string") { - throw Error("opts must be an object with cmd field a string"); - } - // control message - logger.debug("lean command", JSON.stringify(opts)); - switch (opts.cmd) { - case "info": - assert_type("path", opts.path, "string"); - assert_type("line", opts.line, "number"); - assert_type("column", opts.column, "number"); - const r = (await the_lean_server.info(opts.path, opts.line, opts.column)) - .record; - return r ? r : {}; - - // get server version - case "version": - return await the_lean_server.version(); - - // kill the LEAN server. - // this can help with, e.g., updating the LEAN_PATH - case "kill": - return the_lean_server.kill(); - - case "restart": - return await the_lean_server.restart(); - - case "complete": - assert_type("path", opts.path, "string"); - assert_type("line", opts.line, "number"); - assert_type("column", opts.column, "number"); - const complete = await the_lean_server.complete( - opts.path, - opts.line, - opts.column, - opts.skipCompletions - ); - if (complete == null || complete.completions == null) { - return []; - } - // delete the source fields -- they are LARGE and not used at all in the UI. - for (const c of complete.completions) { - delete (c as any).source; // cast because of mistake in upstream type def. sigh. - } - /* - complete.completions.sort(function(a, b): number { - if (a.text == null || b.text == null) { - // satisfy typescript null checks; shouldn't happen. - return 0; - } - return cmp(a.text.toLowerCase(), b.text.toLowerCase()); - }); - */ - return complete.completions; - - default: - throw Error(`unknown cmd ${opts.cmd}`); - } -} diff --git a/src/packages/project/nats/api/editor.ts b/src/packages/project/nats/api/editor.ts deleted file mode 100644 index 4182062dd2..0000000000 --- a/src/packages/project/nats/api/editor.ts +++ /dev/null @@ -1,7 +0,0 @@ -export { jupyter_strip_notebook as jupyterStripNotebook } from "@cocalc/jupyter/nbgrader/jupyter-parse"; -export { jupyter_run_notebook as jupyterRunNotebook } from "@cocalc/jupyter/nbgrader/jupyter-run"; -export { nbconvert as jupyterNbconvert } from "../../jupyter/convert"; -export { run_formatter_string as formatterString } from "../../formatters"; -export { logo as jupyterKernelLogo } from "@cocalc/jupyter/kernel/logo"; -export { get_kernel_data as jupyterKernels } from "@cocalc/jupyter/kernel/kernel-data"; -export { newFile } from "@cocalc/backend/misc/new-file"; diff --git a/src/packages/project/nats/api/index.ts b/src/packages/project/nats/api/index.ts deleted file mode 100644 index cfd87f2cbf..0000000000 --- a/src/packages/project/nats/api/index.ts +++ /dev/null @@ -1,216 +0,0 @@ -/* - -DEVELOPMENT: - -How to do development (so in a dev project doing cc-in-cc dev). - -0. From the browser, terminate this api server running in the project: - - await cc.client.nats_client.projectApi(cc.current()).system.terminate({service:'api'}) - -1. Create a file project-env.sh as explained in projects/nats/README.md, which defines these environment variables (your values will be different): - - export COCALC_PROJECT_ID="00847397-d6a8-4cb0-96a8-6ef64ac3e6cf" - export COCALC_USERNAME=`echo $COCALC_PROJECT_ID | tr -d '-'` - export HOME="/projects/6b851643-360e-435e-b87e-f9a6ab64a8b1/cocalc/src/data/projects/$COCALC_PROJECT_ID" - export DATA=$HOME/.smc - - # optional for more flexibility - export API_KEY=sk-OUwxAN8d0n7Ecd48000055 - export COMPUTE_SERVER_ID=0 - - # optional for more logging - export DEBUG=cocalc:* - export DEBUG_CONSOLE=yes - -If API_KEY is a project-wide API key, then you can change COCALC_PROJECT_ID however you want -and don't have to worry about whether the project is running or the project secret key changing -when the project is restarted. - -2. Then do this: - - $ . project-env.sh - $ node - ... - > require("@cocalc/project/nats/api/index").init() - -You can then easily be able to grab some state, e.g., by writing this in any cocalc code, -rebuilding and restarting: - - global.x = {...} - -Remember, if you don't set API_KEY, then the project MUST be running so that the secret token in $HOME/.smc/secret_token is valid. - -3. Use the browser to see the project is on nats and works: - - a = cc.client.nats_client.projectApi({project_id:'81e0c408-ac65-4114-bad5-5f4b6539bd0e'}); - await a.system.ping(); - await a.system.exec({command:'echo $COCALC_PROJECT_ID'}); - -*/ - -import { JSONCodec } from "nats"; -import getLogger from "@cocalc/backend/logger"; -import { type ProjectApi } from "@cocalc/nats/project-api"; -import getConnection from "@cocalc/project/nats/connection"; -import { getSubject } from "../names"; -import { terminate as terminateOpenFiles } from "@cocalc/project/nats/open-files"; -import { close as closeListings } from "@cocalc/project/nats/listings"; -import { Svcm } from "@nats-io/services"; -import { compute_server_id, project_id } from "@cocalc/project/data"; -import { close as closeFilesRead } from "@cocalc/project/nats/files/read"; -import { close as closeFilesWrite } from "@cocalc/project/nats/files/write"; -import { delay } from "awaiting"; -import { waitUntilConnected } from "@cocalc/nats/util"; - -const MONITOR_INTERVAL = 30000; - -const logger = getLogger("project:nats:api"); -const jc = JSONCodec(); - -export function init() { - mainLoop(); -} - -let terminate = false; -export async function mainLoop() { - let d = 3000; - let lastStart = 0; - while (!terminate) { - try { - lastStart = Date.now(); - await serve(); - logger.debug("project nats api service ended"); - } catch (err) { - logger.debug(`project nats api service error -- ${err}`); - if (Date.now() - lastStart >= 30000) { - // it ran for a while, so no delay - logger.debug(`will restart immediately`); - d = 3000; - } else { - // it crashed quickly, so delay! - d = Math.min(20000, d * 1.25 + Math.random()); - logger.debug(`will restart in ${d}ms`); - await delay(d); - } - } - } -} - -async function serve() { - logger.debug("serve: create project nats api service"); - await waitUntilConnected(); - const nc = await getConnection(); - const subject = getSubject({ service: "api" }); - // @ts-ignore - const svcm = new Svcm(nc); - const name = `project-${project_id}`; - logger.debug(`serve: creating API microservice ${name}`); - await waitUntilConnected(); - const service = await svcm.add({ - name, - version: "0.1.0", - description: `CoCalc ${compute_server_id ? "Compute Server" : "Project"}`, - }); - const api = service.addEndpoint("api", { subject }); - serviceMonitor({ api, subject, nc }); - logger.debug(`serve: subscribed to subject='${subject}'`); - await listen(api, subject); -} - -async function serviceMonitor({ nc, api, subject }) { - while (true) { - logger.debug(`serviceMonitor: waiting ${MONITOR_INTERVAL}ms`); - await delay(MONITOR_INTERVAL); - try { - await waitUntilConnected(); - await nc.request(subject, jc.encode({ name: "ping" }), { - timeout: 7500, - }); - logger.debug("serviceMonitor: ping succeeded"); - } catch (err) { - logger.debug( - `serviceMonitor: ping failed, so restarting service -- ${err}`, - ); - api.stop(); - return; - } - } -} - -async function listen(api, subject) { - for await (const mesg of api) { - const request = jc.decode(mesg.data) ?? ({} as any); - // logger.debug("got message", request); - if (request.name == "system.terminate") { - // TODO: should be part of handleApiRequest below, but done differently because - // one case halts this loop - const { service } = request.args[0] ?? {}; - if (service == "open-files") { - terminateOpenFiles(); - mesg.respond(jc.encode({ status: "terminated", service })); - continue; - } else if (service == "listings") { - closeListings(); - mesg.respond(jc.encode({ status: "terminated", service })); - continue; - } else if (service == "files:read") { - await closeFilesRead(); - mesg.respond(jc.encode({ status: "terminated", service })); - continue; - } else if (service == "files:write") { - await closeFilesWrite(); - mesg.respond(jc.encode({ status: "terminated", service })); - continue; - } else if (service == "api") { - // special hook so admin can terminate handling. This is useful for development. - terminate = true; - console.warn("TERMINATING listening on ", subject); - logger.debug("TERMINATING listening on ", subject); - mesg.respond(jc.encode({ status: "terminated", service })); - api.stop(); - return; - } else { - mesg.respond(jc.encode({ error: `Unknown service ${service}` })); - } - } else { - handleApiRequest(request, mesg); - } - } -} - -async function handleApiRequest(request, mesg) { - let resp; - const { name, args } = request as any; - if (name == "ping") { - resp = "pong"; - } else { - try { - // logger.debug("handling project.api request:", { name }); - resp = (await getResponse({ name, args })) ?? null; - } catch (err) { - logger.debug(`project.api request err = ${err}`, { name }); - resp = { error: `${err}` }; - } - } - mesg.respond(jc.encode(resp)); -} - -import * as system from "./system"; -import * as editor from "./editor"; -import * as sync from "./sync"; - -export const projectApi: ProjectApi = { - system, - editor, - sync, -}; - -async function getResponse({ name, args }) { - const [group, functionName] = name.split("."); - const f = projectApi[group]?.[functionName]; - if (f == null) { - throw Error(`unknown function '${name}'`); - } - return await f(...args); -} diff --git a/src/packages/project/nats/connection.ts b/src/packages/project/nats/connection.ts deleted file mode 100644 index 7a2455dcc8..0000000000 --- a/src/packages/project/nats/connection.ts +++ /dev/null @@ -1,51 +0,0 @@ -/* -Create a unique connection to the nats server. The CONNECT_OPTIONS are such that -the connection should never end up in the closed state. - -If the environment variable NATS_SERVER is set, this tries to connect to that server. -The server should be of this form for a Websocket server - - ws://hostname:port/path/to/nats - -or this for a TCP server: ip-address:port. -That said, for projects and compute servers, **always use a WebSocket**, -since the connection goes through node-http-proxy, so we have more control (e.g., -can kill it), and we also don't have to expose NATS directly to any untrusted -servers. -*/ - -import getConnection, { - setConnectionOptions, -} from "@cocalc/backend/nats/persistent-connection"; -import { getLogger } from "@cocalc/project/logger"; -import { apiKey, natsWebsocketServer } from "@cocalc/backend/data"; -import { inboxPrefix as getInboxPrefix } from "@cocalc/nats/names"; -import { project_id } from "@cocalc/project/data"; -import secretToken from "@cocalc/project/servers/secret-token"; -import * as kucalc from "../kucalc"; - -export default getConnection; - -const logger = getLogger("project:nats:connection"); - -function getServers() { - if (kucalc.IN_KUCALC) { - return "nats-server"; - } - if (process.env.NATS_SERVER) { - return process.env.NATS_SERVER; - } else { - return natsWebsocketServer; - } -} - -setConnectionOptions(async () => { - logger.debug("setting connection options"); - return { - inboxPrefix: getInboxPrefix({ project_id }), - servers: getServers(), - name: JSON.stringify({ project_id }), - user: `project-${project_id}`, - token: apiKey ? apiKey : await secretToken(), - }; -}); diff --git a/src/packages/project/nats/env.ts b/src/packages/project/nats/env.ts deleted file mode 100644 index ace0b589d6..0000000000 --- a/src/packages/project/nats/env.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { sha1 } from "@cocalc/backend/sha1"; -import getConnection from "./connection"; -import { JSONCodec } from "nats"; -import { setNatsClient } from "@cocalc/nats/client"; -import { compute_server_id, project_id } from "@cocalc/project/data"; -import { getLogger } from "@cocalc/project/logger"; - -const jc = JSONCodec(); -export async function getEnv() { - const nc = await getConnection(); - return { sha1, nc, jc }; -} - -export function init() { - setNatsClient({ - getNatsEnv: getEnv, - project_id, - compute_server_id, - getLogger, - }); -} -init(); diff --git a/src/packages/project/nats/pubsub.ts b/src/packages/project/nats/pubsub.ts deleted file mode 100644 index e02a26a3ab..0000000000 --- a/src/packages/project/nats/pubsub.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { getEnv } from "./env"; -import { PubSub } from "@cocalc/nats/sync/pubsub"; -import { project_id } from "@cocalc/project/data"; - -export default async function pubsub({ - path, - name, -}: { - path?: string; - name: string; -}) { - return new PubSub({ env: await getEnv(), project_id, path, name }); -} diff --git a/src/packages/project/nats/sync.ts b/src/packages/project/nats/sync.ts deleted file mode 100644 index bd7d1aa013..0000000000 --- a/src/packages/project/nats/sync.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { stream as createStream, type Stream } from "@cocalc/nats/sync/stream"; -import { - dstream as createDstream, - type DStream, -} from "@cocalc/nats/sync/dstream"; -import { kv as createKV, type KV } from "@cocalc/nats/sync/kv"; -import { dkv as createDKV, type DKV } from "@cocalc/nats/sync/dkv"; -import { dko as createDKO, type DKO } from "@cocalc/nats/sync/dko"; -import { getEnv } from "./env"; -import { project_id } from "@cocalc/project/data"; -import { - createOpenFiles, - type OpenFiles, - Entry as OpenFileEntry, -} from "@cocalc/nats/sync/open-files"; -import { - inventory as createInventory, - type Inventory, -} from "@cocalc/nats/sync/inventory"; - -export type { Stream, DStream, KV, DKV, OpenFiles, OpenFileEntry }; - -export async function stream(opts): Promise> { - return await createStream({ project_id, env: await getEnv(), ...opts }); -} - -export async function dstream(opts): Promise> { - return await createDstream({ project_id, env: await getEnv(), ...opts }); -} - -export async function kv(opts): Promise> { - return await createKV({ project_id, env: await getEnv(), ...opts }); -} - -export async function dkv(opts): Promise> { - return await createDKV({ project_id, env: await getEnv(), ...opts }); -} - -export async function dko(opts): Promise> { - return await createDKO({ project_id, env: await getEnv(), ...opts }); -} - -export async function openFiles(): Promise { - return await createOpenFiles({ project_id }); -} - -export async function inventory(): Promise { - return await createInventory({ project_id }); -} diff --git a/src/packages/project/nats/synctable.ts b/src/packages/project/nats/synctable.ts deleted file mode 100644 index a06b4253ea..0000000000 --- a/src/packages/project/nats/synctable.ts +++ /dev/null @@ -1,38 +0,0 @@ -import getConnection from "./connection"; -import { project_id } from "@cocalc/project/data"; -import { JSONCodec } from "nats"; -import { - createSyncTable, - type NatsSyncTable, -} from "@cocalc/nats/sync/synctable"; -import { parse_query } from "@cocalc/sync/table/util"; -import { keys } from "lodash"; -import type { NatsSyncTableFunction } from "@cocalc/nats/sync/synctable"; - -const jc = JSONCodec(); - -const synctable: NatsSyncTableFunction = async ( - query, - options?, -): Promise => { - const nc = await getConnection(); - query = parse_query(query); - const table = keys(query)[0]; - const obj = options?.obj; - if (obj != null) { - for (const k in obj) { - query[table][0][k] = obj[k]; - } - } - query[table][0].project_id = project_id; - const s = createSyncTable({ - project_id, - ...options, - query, - env: { jc, nc }, - }); - await s.init(); - return s; -}; - -export default synctable; diff --git a/src/packages/project/package.json b/src/packages/project/package.json index 31c941df02..cbafebc9c6 100644 --- a/src/packages/project/package.json +++ b/src/packages/project/package.json @@ -4,8 +4,10 @@ "description": "CoCalc: project daemon", "exports": { "./named-servers": "./dist/named-servers/index.js", - "./nats": "./dist/nats/index.js", - "./*": "./dist/*.js" + "./conat": "./dist/conat/index.js", + "./conat/terminal": "./dist/conat/terminal/index.js", + "./*": "./dist/*.js", + "./project-info": "./dist/project-info/index.js" }, "keywords": [ "python", @@ -21,57 +23,41 @@ "dependencies": { "@cocalc/backend": "workspace:*", "@cocalc/comm": "workspace:*", + "@cocalc/conat": "workspace:*", "@cocalc/jupyter": "workspace:*", - "@cocalc/nats": "workspace:*", "@cocalc/primus-multiplex": "^1.1.0", "@cocalc/primus-responder": "^1.0.5", "@cocalc/project": "workspace:*", "@cocalc/sync": "workspace:*", "@cocalc/sync-client": "workspace:*", "@cocalc/sync-fs": "workspace:*", - "@cocalc/terminal": "workspace:*", "@cocalc/util": "workspace:*", "@lydell/node-pty": "^1.1.0", - "@nats-io/jetstream": "3.0.0", - "@nats-io/kv": "3.0.0", - "@nats-io/services": "3.0.0", - "@nteract/messaging": "^7.0.20", + "@types/jest": "^29.5.14", "awaiting": "^3.0.0", "body-parser": "^1.20.3", "commander": "^7.2.0", "compression": "^1.7.4", "daemonize-process": "^3.0.0", - "debug": "^4.4.0", "diskusage": "^1.1.3", - "expect": "^26.6.2", "express": "^4.21.2", "express-rate-limit": "^7.4.0", "get-port": "^5.1.1", - "googlediff": "^0.1.0", - "json-stable-stringify": "^1.0.1", - "jupyter-paths": "^2.0.3", - "lean-client-js-node": "^1.2.12", "lodash": "^4.17.21", - "lru-cache": "^7.18.3", - "nats": "^2.29.3", - "pidusage": "^1.2.0", "prettier": "^3.0.2", "primus": "^8.0.9", "prom-client": "^13.0.0", "rimraf": "^5.0.5", "temp": "^0.9.4", "tmp": "0.0.33", - "uglify-js": "^3.14.1", "uuid": "^8.3.2", "websocket-sftp": "^0.8.4", "which": "^2.0.2", - "ws": "^8.18.0", - "zeromq": "^5.2.8" + "ws": "^8.18.0" }, "devDependencies": { "@types/body-parser": "^1.19.5", "@types/express": "^4.17.21", - "@types/jquery": "^3.5.5", "@types/lodash": "^4.14.202", "@types/node": "^18.16.14", "@types/primus": "^7.3.9", @@ -83,6 +69,7 @@ "build": "../node_modules/.bin/tsc --build", "tsc": "../node_modules/.bin/tsc --watch --pretty --preserveWatchOutput", "test": "COCALC_PROJECT_ID=812abe34-a382-4bd1-9071-29b6f4334f03 COCALC_USERNAME=user pnpm exec jest", + "depcheck": "pnpx depcheck", "prepublishOnly": "pnpm test", "clean": "rm -rf dist" }, diff --git a/src/packages/project/port_manager.ts b/src/packages/project/port_manager.ts index 9a5d196809..256a032c1b 100644 --- a/src/packages/project/port_manager.ts +++ b/src/packages/project/port_manager.ts @@ -4,43 +4,34 @@ */ import { readFile } from "node:fs/promises"; - -import { abspath } from "@cocalc/backend/misc_node"; +import { sageServerPaths } from "@cocalc/project/data"; type Type = "sage"; /* -The port_manager manages the ports for the various servers. - -It reads the port from memory or from disk and returns it. +The port_manager manages the ports for the sage worksheet server. */ -const { SMC } = process.env; - -function port_file(type: Type): string { - return `${SMC}/${type}_server/${type}_server.port`; -} - // a local cache const ports: { [type in Type]?: number } = {}; -export async function get_port(type: Type): Promise { +export async function get_port(type: Type = "sage"): Promise { const val = ports[type]; if (val != null) { return val; } else { - const content = await readFile(abspath(port_file(type))); + const content = await readFile(sageServerPaths.port); try { const val = parseInt(content.toString()); ports[type] = val; return val; - } catch (e) { - throw new Error(`${type}_server port file corrupted`); + } catch (err) { + throw new Error(`${type}_server port file corrupted -- ${err}`); } } } -export function forget_port(type: Type) { +export function forget_port(type: Type = "sage") { if (ports[type] != null) { delete ports[type]; } diff --git a/src/packages/project/print_to_pdf.ts b/src/packages/project/print_to_pdf.ts index c6b689db61..da6c5df385 100644 --- a/src/packages/project/print_to_pdf.ts +++ b/src/packages/project/print_to_pdf.ts @@ -10,14 +10,11 @@ import { unlink, writeFile } from "node:fs/promises"; import { path as temp_path } from "temp"; -import { execute_code } from "@cocalc/backend/misc_node"; +import { executeCode } from "@cocalc/backend/execute-code"; import { CoCalcSocket } from "@cocalc/backend/tcp/enable-messaging-protocol"; import * as message from "@cocalc/util/message"; import { defaults, filename_extension, required } from "@cocalc/util/misc"; -import { getLogger } from "@cocalc/backend/logger"; -const winston = getLogger("print-to-pdf"); - interface SagewsPrintOpts { path: string; outfile: string; @@ -31,7 +28,7 @@ interface SagewsPrintOpts { timeout?: number; } -async function print_sagews(opts: SagewsPrintOpts) { +export async function printSageWS(opts: SagewsPrintOpts) { opts = defaults(opts, { path: required, outfile: required, @@ -65,8 +62,6 @@ async function print_sagews(opts: SagewsPrintOpts) { args = args.concat(["--base_url", opts.base_url]); } - let err: Error | undefined = undefined; - try { if (opts.extra_data != null) { extra_data_file = temp_path() + ".json"; @@ -77,33 +72,17 @@ async function print_sagews(opts: SagewsPrintOpts) { } // run the converter script - await new Promise((resolve, reject) => { - execute_code({ - command: "smc-sagews2pdf", - args, - err_on_exit: true, - bash: false, - timeout: opts.timeout, - cb: (err) => { - if (err) { - winston.debug(`Issue running smc-sagews2pdf: ${err}`); - reject(err); - } else { - resolve(); - } - }, - }); + await executeCode({ + command: "smc-sagews2pdf", + args, + err_on_exit: true, + bash: false, + timeout: opts.timeout, }); - } catch (err) { - err = err; - } - - if (extra_data_file != null) { - unlink(extra_data_file); // no need to wait for completion before calling opts.cb - } - - if (err) { - throw err; + } finally { + if (extra_data_file != null) { + unlink(extra_data_file); // no need to wait + } } } @@ -119,7 +98,7 @@ export async function print_to_pdf(socket: CoCalcSocket, mesg) { try { switch (ext) { case "sagews": - await print_sagews({ + await printSageWS({ path: mesg.path, outfile: pdf, title: mesg.options.title, @@ -139,12 +118,12 @@ export async function print_to_pdf(socket: CoCalcSocket, mesg) { // all good return socket.write_mesg( "json", - message.printed_to_pdf({ id: mesg.id, path: pdf }) + message.printed_to_pdf({ id: mesg.id, path: pdf }), ); } catch (err) { return socket.write_mesg( "json", - message.error({ id: mesg.id, error: err }) + message.error({ id: mesg.id, error: err }), ); } } diff --git a/src/packages/project/project-info/index.ts b/src/packages/project/project-info/index.ts index 2369ceda9c..c2f0c3e436 100644 --- a/src/packages/project/project-info/index.ts +++ b/src/packages/project/project-info/index.ts @@ -3,5 +3,5 @@ * License: MS-RSL – see LICENSE.md for details */ -export { project_info_ws, get_ProjectInfoServer } from "./project-info"; +export { get_ProjectInfoServer } from "./project-info"; export { ProjectInfoServer } from "./server" diff --git a/src/packages/project/project-info/project-info.ts b/src/packages/project/project-info/project-info.ts index cb897d01f5..d4a1fc0b34 100644 --- a/src/packages/project/project-info/project-info.ts +++ b/src/packages/project/project-info/project-info.ts @@ -7,61 +7,26 @@ Project information */ -import { ProjectInfoCmds } from "@cocalc/util/types/project-info/types"; import { ProjectInfoServer } from "./server"; -import { exec } from "./utils"; +import { createService } from "@cocalc/conat/project/project-info"; +import { project_id, compute_server_id } from "@cocalc/project/data"; // singleton, we instantiate it when we need it -let _info: ProjectInfoServer | undefined = undefined; +let info: ProjectInfoServer | null = null; +let service: any = null; export function get_ProjectInfoServer(): ProjectInfoServer { - if (_info != null) return _info; - _info = new ProjectInfoServer(); - return _info; -} - -export async function project_info_ws( - primus: any, - logger: { debug: Function } -): Promise { - const L = (...msg) => logger.debug("project_info:", ...msg); - const name = `project_info`; - const channel = primus.channel(name); - - function deregister(spark) { - L(`deregistering ${spark.id}`); + if (info != null) { + return info; } + info = new ProjectInfoServer(); + service = createService({ infoServer: info, project_id, compute_server_id }); - channel.on("connection", function (spark: any): void { - // Now handle the connection - L(`channel: new connection from ${spark.address.ip} -- ${spark.id}`); - - function close(type) { - L(`event ${type}: deregistering`); - deregister(spark); - } - - spark.on("close", () => close("close")); - spark.on("end", () => close("end")); - spark.on("data", function (data: ProjectInfoCmds) { - // we assume only ProjectInfoCmds should come in, but better check what this is - if (typeof data === "object") { - switch (data.cmd) { - case "signal": - L(`Signal ${data.signal} from ${spark.id} for pids: ${data.pids}`); - exec(`kill -s ${data.signal ?? 15} ${data.pids.join(" ")}`); - break; - default: - throw Error("WARNING: unknown command -- " + data.cmd); - } - } - }); - }); - - channel.on("disconnection", function (spark: any): void { - L(`channel: disconnection from ${spark.address.ip} -- ${spark.id}`); - deregister(spark); - }); + return info; +} - return name; +export function close() { + service?.close(); + info?.close(); + info = service = null; } diff --git a/src/packages/project/project-info/server.ts b/src/packages/project/project-info/server.ts index 4a231bfbc6..0661b28e48 100644 --- a/src/packages/project/project-info/server.ts +++ b/src/packages/project/project-info/server.ts @@ -6,6 +6,8 @@ /* Project information server, doing the heavy lifting of telling the client about what's going on in a project. + +This is an event emitter that emits a ProjectInfo object periodically when running. */ import { delay } from "awaiting"; @@ -14,8 +16,8 @@ import { check as df } from "diskusage"; import { EventEmitter } from "node:events"; import { readFile } from "node:fs/promises"; import { ProcessStats } from "@cocalc/backend/process-stats"; -import { pidToPath as terminalPidToPath } from "@cocalc/terminal"; -import { +import { pidToPath as terminalPidToPath } from "@cocalc/project/conat/terminal/manager"; +import type { CGroup, CoCalcInfo, DiskUsage, @@ -24,15 +26,10 @@ import { ProjectInfo, } from "@cocalc/util/types/project-info/types"; import { get_path_for_pid as x11_pid2path } from "../x11/server"; -//import { get_sage_path } from "../sage_session" import { getLogger } from "../logger"; const L = getLogger("project-info:server").debug; -// function is_in_dev_project() { -// return process.env.SMC_LOCAL_HUB_HOME != null; -// } - const bytes2MiB = (bytes) => bytes / (1024 * 1024); export class ProjectInfoServer extends EventEmitter { @@ -226,6 +223,10 @@ export class ProjectInfoServer extends EventEmitter { this.running = false; } + close = () => { + this.stop(); + }; + public async start(): Promise { if (this.running) { this.dbg("project-info/server: already running, cannot be started twice"); diff --git a/src/packages/project/project-setup.ts b/src/packages/project/project-setup.ts index 12fa969b38..7ee53f9ce8 100644 --- a/src/packages/project/project-setup.ts +++ b/src/packages/project/project-setup.ts @@ -135,6 +135,10 @@ export function cleanup(): void { "PATH_COCALC", "COCALC_ROOT", "DEBUG_CONSOLE", + "CONAT_SERVER", + "PORT", + "HISTFILE", + "PROMPT_COMMAND", ]; envrm.forEach((name) => delete process.env[name]); diff --git a/src/packages/project/project-status/index.ts b/src/packages/project/project-status/index.ts deleted file mode 100644 index b30058a77f..0000000000 --- a/src/packages/project/project-status/index.ts +++ /dev/null @@ -1,6 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -export { get_ProjectStatusServer, ProjectStatusServer } from "./server"; diff --git a/src/packages/project/project-status/server.ts b/src/packages/project/project-status/server.ts index 53a7c44703..c0b6e98832 100644 --- a/src/packages/project/project-status/server.ts +++ b/src/packages/project/project-status/server.ts @@ -36,13 +36,15 @@ import { ProjectStatus, } from "@cocalc/comm/project-status/types"; import { cgroup_stats } from "@cocalc/comm/project-status/utils"; +import { createPublisher } from "@cocalc/conat/project/project-status"; +import { compute_server_id, project_id } from "@cocalc/project/data"; // TODO: only return the "next" value, if it is significantly different from "prev" //function threshold(prev?: number, next?: number): number | undefined { // return next; //} -const winston = getLogger("ProjectStatusServer"); +const logger = getLogger("project-status:server"); function quantize(val, order) { const q = Math.round(Math.pow(10, order)); @@ -83,7 +85,7 @@ export class ProjectStatusServer extends EventEmitter { constructor(testing = false) { super(); this.testing = testing; - this.dbg = (...msg) => winston.debug(...msg); + this.dbg = (...msg) => logger.debug(...msg); this.project_info = get_ProjectInfoServer(); } @@ -297,11 +299,7 @@ export class ProjectStatusServer extends EventEmitter { } public async start(): Promise { - if (this.running) { - this.dbg( - "project-status/server: already running, cannot be started twice", - ); - } else { + if (!this.running) { await this._start(); } } @@ -326,12 +324,19 @@ export class ProjectStatusServer extends EventEmitter { } // singleton, we instantiate it when we need it -let _status: ProjectStatusServer | undefined = undefined; +let status: ProjectStatusServer | undefined = undefined; -export function get_ProjectStatusServer(): ProjectStatusServer { - if (_status != null) return _status; - _status = new ProjectStatusServer(); - return _status; +export function init() { + logger.debug("initializing project status server, and enabling publishing"); + if (status == null) { + status = new ProjectStatusServer(); + } + createPublisher({ + projectStatusServer: status, + compute_server_id, + project_id, + }); + status.start(); } // testing: $ ts-node server.ts diff --git a/src/packages/project/project.ts b/src/packages/project/project.ts index b09180216a..98d28f9152 100644 --- a/src/packages/project/project.ts +++ b/src/packages/project/project.ts @@ -3,6 +3,9 @@ * License: MS-RSL – see LICENSE.md for details */ +// in case daemonizing is hiding key output, set this +const DEBUG_DAEMON_OUTPUT = false; + import daemonizeProcess from "daemonize-process"; import { init as initBugCounter } from "./bug-counter"; @@ -33,13 +36,17 @@ function checkEnvVariables() { } export async function main() { - initBugCounter(); - checkEnvVariables(); const options = getOptions(); if (options.daemon) { - logger.info("daemonize the process"); - daemonizeProcess(); + logger.info(`daemonize the process pid=${process.pid}`); + if (DEBUG_DAEMON_OUTPUT) { + daemonizeProcess({ stdio: ["inherit", "inherit", "inherit"] }); + } else { + daemonizeProcess(); + } } + initBugCounter(); + checkEnvVariables(); cleanupEnvironmentVariables(); initKucalc(); // must be after cleanupEnvironmentVariables, since this *adds* custom environment variables. logger.info("main init function"); diff --git a/src/packages/project/sage_restart.ts b/src/packages/project/sage_restart.ts index 2a5a8d654a..d7dfb40fa6 100644 --- a/src/packages/project/sage_restart.ts +++ b/src/packages/project/sage_restart.ts @@ -18,8 +18,8 @@ export const SAGE_SERVER_MAX_STARTUP_TIME_S = 60; let restarting = false; let restarted = 0; // time when we last restarted it -export async function restart_sage_server() { - const dbg = (m) => winston.debug(`restart_sage_server: ${to_json(m)}`); +export async function restartSageServer() { + const dbg = (m) => winston.debug(`restartSageServer: ${to_json(m)}`); if (restarting) { dbg("hit lock"); throw new Error("already restarting sage server"); diff --git a/src/packages/project/sage_session.ts b/src/packages/project/sage_session.ts index c694ba5439..8a9d0e55bc 100644 --- a/src/packages/project/sage_session.ts +++ b/src/packages/project/sage_session.ts @@ -27,11 +27,9 @@ import { import { CB } from "@cocalc/util/types/callback"; import { ISageSession, SageCallOpts } from "@cocalc/util/types/sage"; import { Client } from "./client"; -import { get_sage_socket } from "./sage_socket"; +import { getSageSocket } from "./sage_socket"; -// import { ExecuteCodeOutput } from "@cocalc/util/types/execute-code"; - -const winston = getLogger("sage-session"); +const logger = getLogger("sage-session"); //############################################## // Direct Sage socket session -- used internally in local hub, e.g., to assist CodeMirror editors... @@ -52,10 +50,6 @@ export function sage_session(opts: Readonly): SageSessionType { // compute and cache if not cached; otherwise, get from cache: return (cache[path] = cache[path] ?? new SageSession(opts)); } -// TODO for project-info/server we need a function that returns a path to a sage worksheet for a given PID -//export function get_sage_path(pid) {} -// return path -// } /* Sage Session object @@ -69,59 +63,52 @@ class SageSession implements ISageSession { [key: string]: CB<{ done: boolean; error: string }, any>; } = {}; private _socket: CoCalcSocket | undefined; - public init_socket: () => Promise; constructor(opts: Readonly) { this.dbg = this.dbg.bind(this); - this.close = this.close.bind(this); - this.is_running = this.is_running.bind(this); - this._init_socket = this._init_socket.bind(this); - this.init_socket = reuseInFlight(this._init_socket).bind(this); - this._init_path = this._init_path.bind(this); - this.call = this.call.bind(this); - this._handle_mesg_blob = this._handle_mesg_blob.bind(this); - this._handle_mesg_json = this._handle_mesg_json.bind(this); this.dbg("constructor")(); this._path = opts.path; this._client = opts.client; this._output_cb = {}; } - private dbg(f: string) { + private dbg = (f: string) => { return (m?: string) => - winston.debug(`SageSession(path='${this._path}').${f}: ${m}`); - } + logger.debug(`SageSession(path='${this._path}').${f}: ${m}`); + }; - public close(): void { + close = (): void => { if (this._socket != null) { const pid = this._socket.pid; - if (pid != null) processKill(pid, 9); + if (pid != null) { + processKill(pid, 9); + } + this._socket.end(); + delete this._socket; } - this._socket?.end(); - delete this._socket; for (let id in this._output_cb) { const cb = this._output_cb[id]; cb({ done: true, error: "killed" }); } this._output_cb = {}; delete cache[this._path]; - } + }; // return true if there is a socket connection to a sage server process - is_running(): boolean { + is_running = (): boolean => { return this._socket != null; - } + }; // NOTE: There can be many simultaneous init_socket calls at the same time, // if e.g., the socket doesn't exist and there are a bunch of calls to @call // at the same time. // See https://github.com/sagemathinc/cocalc/issues/3506 // wrapped in reuseInFlight ! - private async _init_socket(): Promise { + init_socket = reuseInFlight(async (): Promise => { const dbg = this.dbg("init_socket()"); dbg(); try { - const socket: CoCalcSocket = await get_sage_socket(); + const socket: CoCalcSocket = await getSageSocket(); dbg("successfully opened a sage session"); this._socket = socket; @@ -153,9 +140,9 @@ class SageSession implements ISageSession { throw err; } } - } + }); - private async _init_path(): Promise { + private _init_path = async (): Promise => { const dbg = this.dbg("_init_path()"); dbg(); return new Promise((resolve, reject) => { @@ -185,9 +172,12 @@ class SageSession implements ISageSession { }, }); }); - } + }; - public async call({ input, cb }: Readonly): Promise { + public call = async ({ + input, + cb, + }: Readonly): Promise => { const dbg = this.dbg("call"); dbg(`input='${trunc(to_json(input), 300)}'`); switch (input.event) { @@ -252,8 +242,8 @@ class SageSession implements ISageSession { cb({ done: true, error: err }); } } - } - private _handle_mesg_blob(mesg: TCPMessage) { + }; + private _handle_mesg_blob = (mesg: TCPMessage) => { const { uuid } = mesg; let { blob } = mesg; const dbg = this.dbg(`_handle_mesg_blob(uuid='${uuid}')`); @@ -283,9 +273,9 @@ class SageSession implements ISageSession { this._socket?.write_mesg("json", resp); }, }); - } + }; - private _handle_mesg_json(mesg: TCPMessage) { + private _handle_mesg_json = (mesg: TCPMessage) => { const dbg = this.dbg("_handle_mesg_json"); dbg(`mesg='${trunc_middle(to_json(mesg), 400)}'`); if (mesg == null) return; // should not happen @@ -304,5 +294,5 @@ class SageSession implements ISageSession { } cb(mesg); } - } + }; } diff --git a/src/packages/project/sage_socket.ts b/src/packages/project/sage_socket.ts index 60e98cc82e..c476dd52fc 100644 --- a/src/packages/project/sage_socket.ts +++ b/src/packages/project/sage_socket.ts @@ -4,104 +4,96 @@ */ import { getLogger } from "@cocalc/backend/logger"; +import { secretToken } from "@cocalc/project/data"; import { enable_mesg } from "@cocalc/backend/misc_node"; import { CoCalcSocket } from "@cocalc/backend/tcp/enable-messaging-protocol"; import { connectToLockedSocket } from "@cocalc/backend/tcp/locked-socket"; import * as message from "@cocalc/util/message"; -import { retry_until_success } from "@cocalc/util/misc"; import * as common from "./common"; import { forget_port, get_port } from "./port_manager"; import { SAGE_SERVER_MAX_STARTUP_TIME_S, - restart_sage_server, + restartSageServer, } from "./sage_restart"; -import { getSecretToken } from "./servers/secret-token"; -import { CB } from "@cocalc/util/types/callback"; +import { until, once } from "@cocalc/util/async-utils"; -const winston = getLogger("sage-socket"); +const logger = getLogger("get-sage-socket"); // Get a new connection to the Sage server. If the server // isn't running, e.g., it was killed due to running out of memory, // attempt to restart it and try to connect. -export async function get_sage_socket(): Promise { - let socket: CoCalcSocket | undefined; - const try_to_connect = async (cb: CB) => { - try { - socket = await _get_sage_socket(); - cb(); - } catch (err) { - // Failed for some reason: try to restart one time, then try again. - // We do this because the Sage server can easily get killed due to out of memory conditions. - // But we don't constantly try to restart the server, since it can easily fail to start if - // there is something wrong with a local Sage install. - // Note that restarting the sage server doesn't impact currently running worksheets (they - // have their own process that isn't killed). +export async function getSageSocket(): Promise { + let socket: CoCalcSocket | undefined = undefined; + await until( + async () => { try { - await restart_sage_server(); - // success at restarting sage server: *IMMEDIATELY* try to connect - socket = await _get_sage_socket(); - cb(); + socket = await _getSageSocket(); + return true; } catch (err) { - // won't actually try to restart if called recently. - cb(err); - } - } - }; - - return new Promise((resolve, reject) => { - retry_until_success({ - f: try_to_connect, - start_delay: 50, - max_delay: 5000, - factor: 1.5, - max_time: SAGE_SERVER_MAX_STARTUP_TIME_S * 1000, - log(m) { - winston.debug(`get_sage_socket: ${m}`); - }, - cb(err) { - if (socket == null) { - reject("failed to get sage socket"); - } else if (err) { - reject(err); - } else { - resolve(socket); + logger.debug( + `unable to get sage socket, so restarting sage server - ${err}`, + ); + // Failed for some reason: try to restart one time, then try again. + // We do this because the Sage server can easily get killed due to out of memory conditions. + // But we don't constantly try to restart the server, since it can easily fail to start if + // there is something wrong with a local Sage install. + // Note that restarting the sage server doesn't impact currently running worksheets (they + // have their own process that isn't killed). + try { + // starting the sage server can also easily fail, so must be in the try + await restartSageServer(); + // get socket immediately -- don't want to wait up to ~5s! + socket = await _getSageSocket(); + return true; + } catch (err) { + logger.debug( + `error restarting sage server or getting socket -- ${err}`, + ); } - }, - }); - }); + return false; + } + }, + { + start: 1000, + max: 5000, + decay: 1.5, + timeout: SAGE_SERVER_MAX_STARTUP_TIME_S * 1000, + }, + ); + if (socket === undefined) { + throw Error("bug"); + } + return socket; } -async function _get_sage_socket(): Promise { - winston.debug("get sage server port"); +async function _getSageSocket(): Promise { + logger.debug("get sage server port"); const port = await get_port("sage"); - winston.debug("get and unlock socket"); - if (port == null) throw new Error("port is null"); + logger.debug(`get and unlock socket on port ${port}`); + if (!port) { + throw new Error("port is not set"); + } try { - const sage_socket: CoCalcSocket | undefined = await connectToLockedSocket({ + const sage_socket: CoCalcSocket = await connectToLockedSocket({ port, - token: getSecretToken(), + token: secretToken, }); - winston.debug("Successfully unlocked a sage session connection."); + logger.debug("Successfully unlocked a sage session connection."); - winston.debug("request sage session from server."); + logger.debug("request sage session from server."); enable_mesg(sage_socket); sage_socket.write_mesg("json", message.start_session({ type: "sage" })); - winston.debug( + logger.debug( "Waiting to read one JSON message back, which will describe the session....", ); - // TODO: couldn't this just hang forever :-( - return new Promise((resolve) => { - sage_socket.once("mesg", (_type, desc) => { - winston.debug( - `Got message back from Sage server: ${common.json(desc)}`, - ); - sage_socket.pid = desc.pid; - resolve(sage_socket); - }); - }); - } catch (err2) { + const [_type, desc] = await once(sage_socket, "mesg", 30000); + logger.debug(`Got message back from Sage server: ${common.json(desc)}`); + sage_socket.pid = desc.pid; + return sage_socket; + } catch (err) { forget_port("sage"); - const msg = `_new_session: sage session denied connection: ${err2}`; - throw new Error(msg); + const msg = `_new_session: sage session denied connection: ${err}`; + logger.debug(`Failed to connect -- ${msg}`); + throw Error(msg); } } diff --git a/src/packages/project/secret-token.ts b/src/packages/project/secret-token.ts new file mode 100644 index 0000000000..d26042630d --- /dev/null +++ b/src/packages/project/secret-token.ts @@ -0,0 +1,64 @@ +/* +THE SECRET TOKEN + +There is a column secret_token in the postgresql projects table. That token is +generated by the server and must be made available by the hub to the project at startup, +so the hubs can connect to the project. It is also used internally to secure some +communications (e.g., sage worksheets). The secret token must be written +to a file whose path is either $COCALC_SECRET_TOKEN *or* $DATA/secret-token. + +For a compute server, hubs do not connect to it and shouldn't be able to; +instead compute servers connect to cocalc. In that case the secret token +will always be set to a random value on startup, and used only for internal +communications. +*/ + +import { readFileSync } from "fs"; +import { getLogger } from "./logger"; +import { join } from "path"; +import { data } from "@cocalc/backend/data"; +import { compute_server_id } from "./data"; +import { secureRandomStringSync } from "@cocalc/backend/misc"; + +const logger = getLogger("data"); + +// either this is set to something valid by the code below, or the process exits with an error. +export let secretToken: string = ""; + +function init() { + if (compute_server_id) { + // it's a compute server, so we always set secret token to a random value. + secretToken = secureRandomStringSync(32); + return; + } + // not a compute server -- read from file + try { + logger.debug(`COCALC_SECRET_TOKEN = ${process.env.COCALC_SECRET_TOKEN}`); + const secretTokenPath = + process.env.COCALC_SECRET_TOKEN ?? join(data, "secret-token"); + try { + secretToken = readFileSync(secretTokenPath).toString(); + } catch (err) { + throw Error( + `Failed to read the project's secret token from '${secretTokenPath} -- ${err}.`, + ); + } + if (!secretToken || secretToken.length < 16) { + throw Error( + `secret token read from file ${secretTokenPath} must be defined and at least 16 characters, but secretToken?.length=${secretToken?.length}`, + ); + } + logger.debug("Successfully initialized project secret_token"); + } catch (err) { + console.trace(err); + const mesg = `The secret token must be in the path given by COCALC_SECRET_TOKEN or at '${join(data, "secret-token")}'. There is something wrong with the setup of this project. ${err}`; + logger.debug(mesg); + console.trace(mesg); + setTimeout(() => { + // git the process a chance to output the errors and logs above before actually terminating. + process.exit(1); + }, 2000); + } +} + +init(); diff --git a/src/packages/project/servers/hub/handle-message.ts b/src/packages/project/servers/hub/handle-message.ts index eaa864eafa..f9c13c23d7 100644 --- a/src/packages/project/servers/hub/handle-message.ts +++ b/src/packages/project/servers/hub/handle-message.ts @@ -29,6 +29,7 @@ import { version } from "@cocalc/util/smc-version"; import { Message } from "./types"; import writeTextFileToProject from "./write-text-file-to-project"; import readTextFileFromProject from "./read-text-file-from-project"; +import { jupyter_execute_response } from "@cocalc/util/message"; const logger = getLogger("handle-message-from-hub"); @@ -69,13 +70,17 @@ export default async function handleMessage( case "project_exec": // this is no longer used by web browser clients; however it *is* used by the HTTP api served // by the hub to api key users, so do NOT remove it! E.g., the latex endpoint, the compute - // server, etc., use it. The web browser clients use the websocket api, + // server, etc., use it. The web browser clients use the websocket api. exec_shell_code(socket, mesg); return; case "jupyter_execute": try { - await jupyterExecute(socket, mesg); + const outputs = await jupyterExecute(mesg as any); + socket.write_mesg( + "json", + jupyter_execute_response({ id: mesg.id, output: outputs }), + ); } catch (err) { socket.write_mesg( "json", diff --git a/src/packages/project/servers/hub/tcp-server.ts b/src/packages/project/servers/hub/tcp-server.ts index 7c26ce195a..2385f5bcf3 100644 --- a/src/packages/project/servers/hub/tcp-server.ts +++ b/src/packages/project/servers/hub/tcp-server.ts @@ -8,14 +8,12 @@ import { writeFile } from "node:fs/promises"; import { createServer } from "node:net"; import * as uuid from "uuid"; - import enableMessagingProtocol, { CoCalcSocket, } from "@cocalc/backend/tcp/enable-messaging-protocol"; import { unlockSocket } from "@cocalc/backend/tcp/locked-socket"; -import { hubPortFile } from "@cocalc/project/data"; +import { hubPortFile, secretToken } from "@cocalc/project/data"; import { getOptions } from "@cocalc/project/init-program"; -import { getSecretToken } from "@cocalc/project/servers/secret-token"; import { once } from "@cocalc/util/async-utils"; import handleMessage from "./handle-message"; import { getClient } from "@cocalc/project/client"; @@ -24,13 +22,6 @@ import { getLogger } from "@cocalc/project/logger"; const winston = getLogger("hub-tcp-server"); export default async function init(): Promise { - const secretToken = getSecretToken(); // could throw if not initialized yet - if (!secretToken || secretToken.length < 16) { - // being extra careful since security - throw Error("secret token must be defined and at least 16 characters"); - return; - } - winston.info("starting tcp server: project <--> hub..."); const server = createServer(handleConnection); const options = getOptions(); @@ -58,10 +49,10 @@ async function handleConnection(socket: CoCalcSocket) { }); try { - await unlockSocket(socket, getSecretToken()); + await unlockSocket(socket, secretToken); } catch (err) { winston.error( - "failed to unlock socket -- ignoring any future messages and closing connection" + "failed to unlock socket -- ignoring any future messages and closing connection", ); socket.destroy(new Error("invalid secret token")); return; diff --git a/src/packages/project/servers/init.ts b/src/packages/project/servers/init.ts index 74ad8fedc5..0293c57ffc 100644 --- a/src/packages/project/servers/init.ts +++ b/src/packages/project/servers/init.ts @@ -6,8 +6,6 @@ /* Initialize both the hub and browser servers. */ import initPidFile from "./pid-file"; -import initSecretToken from "./secret-token"; - import initAPIServer from "@cocalc/project/http-api/server"; import initBrowserServer from "./browser/http-server"; import initHubServer from "./hub/tcp-server"; @@ -18,7 +16,6 @@ const winston = getLogger("init-project-server"); export default async function init() { winston.info("Write pid file to disk."); await initPidFile(); - await initSecretToken(); // must be before servers, since they use this. await initAPIServer(); await initBrowserServer(); await initHubServer(); diff --git a/src/packages/project/servers/secret-token.ts b/src/packages/project/servers/secret-token.ts deleted file mode 100644 index a9afda2a56..0000000000 --- a/src/packages/project/servers/secret-token.ts +++ /dev/null @@ -1,49 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2022 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -/* -Generate the "secret_token" if it does not already exist. -*/ - -import { callback } from "awaiting"; -import { randomBytes } from "crypto"; -import { chmod, readFile, writeFile } from "node:fs/promises"; -import { secretToken as secretTokenPath } from "@cocalc/project/data"; -import { getLogger } from "@cocalc/project/logger"; -const winston = getLogger("secret-token"); - -// We use an n-character cryptographic random token, where n -// is given below. If you want to change this, changing only -// the following line should be safe. -const LENGTH = 128; - -let secretToken: string = ""; // not yet initialized - -async function createSecretToken(): Promise { - winston.info(`creating '${secretTokenPath}'`); - - secretToken = (await callback(randomBytes, LENGTH)).toString("base64"); - await writeFile(secretTokenPath, secretToken); - // set restrictive permissions; shouldn't be necessary - await chmod(secretTokenPath, 0o600); - return secretToken; -} - -export default async function init(): Promise { - try { - winston.info(`checking for secret token in "${secretTokenPath}"`); - secretToken = (await readFile(secretTokenPath)).toString(); - return secretToken; - } catch (err) { - return await createSecretToken(); - } -} - -export function getSecretToken(): string { - if (secretToken == "") { - throw Error("secret token not yet initialized"); - } - return secretToken; -} diff --git a/src/packages/project/sync/compute-server-open-file-tracking.ts b/src/packages/project/sync/compute-server-open-file-tracking.ts deleted file mode 100644 index 1eac8bbe3e..0000000000 --- a/src/packages/project/sync/compute-server-open-file-tracking.ts +++ /dev/null @@ -1,127 +0,0 @@ -/* -Manage the state of open files in the compute servers syncdb sync'd file. - -TODO: terminals aren't handled at all here, since they don't have a syncdoc. -*/ - -import type { SyncDocs } from "./sync-doc"; -import type { SyncDB } from "@cocalc/sync/editor/db/sync"; -import { once } from "@cocalc/util/async-utils"; -import { auxFileToOriginal } from "@cocalc/util/misc"; -import { terminalTracker } from "@cocalc/terminal"; -import { getLogger } from "@cocalc/backend/logger"; -import { syncdbPath, JUPYTER_SYNCDB_EXTENSIONS } from "@cocalc/util/jupyter/names"; - -const log = getLogger("project:sync:compute-file-tracker").debug; - -export default async function computeServerOpenFileTracking( - syncDocs: SyncDocs, - compute: SyncDB, -) { - log("initialize"); - if (compute.get_state() != "ready") { - log("wait for compute server syncdoc to be ready..."); - await once(compute, "ready"); - } - - const getOpenPaths = () => { - const v = syncDocs.getOpenPaths().concat(terminalTracker.getOpenPaths()); - log("getOpenPaths", v); - return new Set(v); - }; - const isOpen = (path: string): boolean => { - return syncDocs.isOpen(path) || terminalTracker.isOpen(path); - }; - - // Initialize -- get all open paths and update syncdb state to reflect this correctly. - const openPaths = getOpenPaths(); - for (const { path, open } of compute.get().toJS()) { - const syncdocPath = computePathToSyncDocPath(path); - const isOpen = openPaths.has(syncdocPath); - log("init ", { path, open, syncdocPath, isOpen }); - if (open != isOpen) { - log("init ", "changing state of ", { path }); - compute.set({ path, open: isOpen }); - } - } - compute.commit(); - - // Watch for files being opened or closed or paths being added/removed from syncdb - const handleOpen = (path: string) => { - log("handleOpen", { path }); - if (compute.get_state() == "closed") { - syncDocs.removeListener("open", handleOpen); - return; - } - // A path was opened. If it is in the syncdb, then mark it as opened there. - const x = compute.get_one({ path: syncDocPathToComputePath(path) }); - if (x != null) { - compute.set({ path: syncDocPathToComputePath(path), open: true }); - compute.commit(); - } - }; - syncDocs.on("open", handleOpen); - terminalTracker.on("open", handleOpen); - - const handleClose = (path: string) => { - log("handleClose", { path }); - if (compute.get_state() == "closed") { - syncDocs.removeListener("open", handleClose); - return; - } - // A path was closed. If it is in the syncdb, then mark it as closed there. - const x = compute.get_one({ path: syncDocPathToComputePath(path) }); - if (x != null) { - compute.set({ path: syncDocPathToComputePath(path), open: false }); - compute.commit(); - } - }; - - syncDocs.on("close", handleClose); - // terminals currently don't get closed, but we include this anyways so - // it will "just work" when we do implement that. - terminalTracker.on("close", handleClose); - - // keys is an immutablejs Set of {path} objects - const handleComputeChange = (keys) => { - // The compute server table that tracks where things should run changed. - // If any path was added to that tabl, make sure its open state is correct. - const keyList = keys.toJS(); - log("handleComputeChange", { keyList }); - let n = 0; - for (const { path } of keyList) { - const x = compute.get_one({ path }); - if (x == null) { - // path was REMOVED - log("handleComputeChange: removed", { path }); - continue; - } - // path was added or changed in some way -- make sure it agrees - const open = isOpen(computePathToSyncDocPath(path)); - if (x.get("open") != open) { - log("handleComputeChange -- making change:", { path, open }); - compute.set({ path, open }); - n += 1; - } - } - if (n > 0) { - compute.commit(); - } - }; - - compute.on("change", handleComputeChange); -} - -function syncDocPathToComputePath(path: string): string { - if (path.endsWith("." + JUPYTER_SYNCDB_EXTENSIONS)) { - return auxFileToOriginal(path); - } - return path; -} - -function computePathToSyncDocPath(path: string): string { - if (path.endsWith(".ipynb")) { - return syncdbPath(path); - } - return path; -} diff --git a/src/packages/project/sync/listings.ts b/src/packages/project/sync/listings.ts deleted file mode 100644 index 796f0a01a6..0000000000 --- a/src/packages/project/sync/listings.ts +++ /dev/null @@ -1,55 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -import { - registerListingsTable as registerListingsTable0, - getListingsTable, -} from "@cocalc/sync/listings"; -import getListing from "@cocalc/backend/get-listing"; -import { Watcher } from "@cocalc/backend/path-watcher"; -import { close_all_syncdocs_in_tree } from "./sync-doc"; -import { getLogger } from "@cocalc/backend/logger"; -import { existsSync } from "fs"; - -const logger = getLogger("project:sync:listings"); -const log = logger.debug; - -export { getListingsTable }; - -export function registerListingsTable(table, query): void { - log("registerListingsTables"); - log("registerListingsTables: query=", query); - const onDeletePath = async (path) => { - // Also we need to close *all* syncdocs that are going to be deleted, - // and wait until closing is done before we return. - await close_all_syncdocs_in_tree(path); - }; - - const createWatcher = (path: string, debounce: number) => - new Watcher(path, { debounce }); - - const { project_id, compute_server_id } = query.listings[0]; - - if (compute_server_id == 0) { - log( - "registerListingsTables -- actually registering since compute_server_id=0", - ); - registerListingsTable0({ - table, - project_id, - compute_server_id, - onDeletePath, - getListing, - createWatcher, - existsSync, - getLogger, - }); - } else { - log( - "registerListingsTables -- NOT implemented since compute_server_id=", - compute_server_id, - ); - } -} diff --git a/src/packages/project/sync/open-synctables.ts b/src/packages/project/sync/open-synctables.ts deleted file mode 100644 index 1490b9c724..0000000000 --- a/src/packages/project/sync/open-synctables.ts +++ /dev/null @@ -1,90 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -/* -All SyncTables that are currently open and being managed in this project. - -*/ - -import { callback } from "awaiting"; -import { SyncTable } from "@cocalc/sync/table"; -import { is_array } from "@cocalc/util/misc"; - -const open_synctables: { [key: string]: SyncTable } = {}; -const wait_for: { [key: string]: Function[] } = {}; - -export function key(query): string { - const table: string = Object.keys(query)[0]; - if (!table) { - throw Error("no table in query"); - } - let c = query[table]; - if (c == null) { - throw Error("invalid query format"); - } - if (is_array(c)) { - c = c[0]; - } - const string_id = c.string_id; - if (string_id == null) { - throw Error( - "open-syncstring-tables is only for queries with a specified string_id field" + - "(patches, cursors, eval_inputs, eval_outputs, etc.): query=" + - JSON.stringify(query) - ); - } - return `${table}.${string_id}`; -} - -export function register_synctable(query: any, synctable: SyncTable): void { - const k = key(query); - open_synctables[k] = synctable; - synctable.on("closed", function () { - delete open_synctables[k]; - }); - if (wait_for[k] != null) { - handle_wait_for(k, synctable); - } -} - -export async function get_synctable(query, client): Promise { - const k = key(query); - const log = client.dbg(`get_synctable(key=${k})`); - log("open_synctables = ", Object.keys(open_synctables)); - log("key=", k, "query=", query); - const s = open_synctables[k]; - if (s != null) { - // easy - already have it. - log("done"); - return s; - } - function f(cb: Function) { - log("f got called"); - add_to_wait_for(k, cb); - } - log("waiting..."); - const synctable = await callback(f); - log(`got the synctable! ${JSON.stringify((synctable as any).query)}`); - return synctable; -} - -function add_to_wait_for(k: string, cb: Function): void { - if (wait_for[k] == null) { - wait_for[k] = [cb]; - } else { - wait_for[k].push(cb); - } -} - -function handle_wait_for(k: string, synctable: SyncTable): void { - if (wait_for[k] == null) { - return; - } - const v: Function[] = wait_for[k]; - delete wait_for[k]; - for (const cb of v) { - cb(undefined, synctable); - } -} diff --git a/src/packages/project/sync/project-info.ts b/src/packages/project/sync/project-info.ts deleted file mode 100644 index c31d0eb2ee..0000000000 --- a/src/packages/project/sync/project-info.ts +++ /dev/null @@ -1,89 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; -import { close } from "@cocalc/util/misc"; -import { SyncTable } from "@cocalc/sync/table"; -import { get_ProjectInfoServer } from "../project-info"; -import { ProjectInfo } from "@cocalc/util/types/project-info/types"; -import { ProjectInfoServer } from "../project-info"; - -class ProjectInfoTable { - private table: SyncTable; - private logger: { debug: Function }; - private project_id: string; - private state: "ready" | "closed" = "ready"; - private readonly publish: (info: ProjectInfo) => Promise; - private readonly info_server: ProjectInfoServer; - - constructor( - table: SyncTable, - logger: { debug: Function }, - project_id: string - ) { - this.project_id = project_id; - this.logger = logger; - this.log("register"); - this.publish = reuseInFlight(this.publish_impl.bind(this)); - this.table = table; - this.table.on("closed", () => this.close()); - // initializing project info server + reacting when it has something to say - this.info_server = get_ProjectInfoServer(); - this.info_server.start(); - this.info_server.on("info", this.publish); - } - - private async publish_impl(info: ProjectInfo): Promise { - if (this.state == "ready" && this.table.get_state() != "closed") { - const next = { project_id: this.project_id, info }; - this.table.set(next, "shallow"); - try { - await this.table.save(); - } catch (err) { - this.log(`error saving ${err}`); - } - } else if (this.log != null) { - this.log( - `ProjectInfoTable state = '${ - this.state - }' and table is '${this.table?.get_state()}'` - ); - } - } - - public close(): void { - this.log("close"); - this.info_server?.off("info", this.publish); - this.table?.close_no_async(); - close(this); - this.state = "closed"; - } - - private log(...args): void { - if (this.logger == null) return; - this.logger.debug("project_info", ...args); - } -} - -let project_info_table: ProjectInfoTable | undefined = undefined; - -export function register_project_info_table( - table: SyncTable, - logger: any, - project_id: string -): void { - logger.debug("register_project_info_table"); - if (project_info_table != null) { - logger.debug( - "register_project_info_table: cleaning up an already existing one" - ); - project_info_table.close(); - } - project_info_table = new ProjectInfoTable(table, logger, project_id); -} - -export function get_project_info_table(): ProjectInfoTable | undefined { - return project_info_table; -} diff --git a/src/packages/project/sync/project-status.ts b/src/packages/project/sync/project-status.ts deleted file mode 100644 index 14fb1a73b8..0000000000 --- a/src/packages/project/sync/project-status.ts +++ /dev/null @@ -1,97 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; -import { close } from "@cocalc/util/misc"; -import { SyncTable } from "@cocalc/sync/table"; -import { - get_ProjectStatusServer, - ProjectStatusServer, -} from "../project-status"; -import { ProjectStatus } from "@cocalc/comm/project-status/types"; - -class ProjectStatusTable { - private table: SyncTable; - private logger: { debug: Function }; - private project_id: string; - private state: "ready" | "closed" = "ready"; - private readonly publish: (status: ProjectStatus) => Promise; - private readonly status_server: ProjectStatusServer; - - constructor( - table: SyncTable, - logger: { debug: Function }, - project_id: string, - ) { - this.status_handler = this.status_handler.bind(this); - this.project_id = project_id; - this.logger = logger; - this.log("register"); - this.publish = reuseInFlight(this.publish_impl.bind(this)); - this.table = table; - this.table.on("closed", () => this.close()); - // initializing project status server + reacting when it has something to say - this.status_server = get_ProjectStatusServer(); - this.status_server.start(); - this.status_server.on("status", this.status_handler); - } - - private status_handler(status): void { - this.log?.("status_server event 'status'", status.timestamp); - this.publish?.(status); - } - - private async publish_impl(status: ProjectStatus): Promise { - if (this.state == "ready" && this.table.get_state() != "closed") { - const next = { project_id: this.project_id, status }; - this.table.set(next, "shallow"); - try { - await this.table.save(); - } catch (err) { - this.log(`error saving ${err}`); - } - } else if (this.log != null) { - this.log( - `ProjectStatusTable '${ - this.state - }' and table is ${this.table?.get_state()}`, - ); - } - } - - public close(): void { - this.log("close"); - this.status_server?.off("status", this.status_handler); - this.table?.close_no_async(); - close(this); - this.state = "closed"; - } - - private log(...args): void { - if (this.logger == null) return; - this.logger.debug("project_status", ...args); - } -} - -let project_status_table: ProjectStatusTable | undefined = undefined; - -export function register_project_status_table( - table: SyncTable, - logger: any, - project_id: string, -): void { - logger.debug("register_project_status_table"); - if (project_status_table != null) { - logger.debug( - "register_project_status_table: cleaning up an already existing one", - ); - project_status_table.close(); - } - project_status_table = new ProjectStatusTable(table, logger, project_id); -} - -export function get_project_status_table(): ProjectStatusTable | undefined { - return project_status_table; -} diff --git a/src/packages/project/sync/server.ts b/src/packages/project/sync/server.ts deleted file mode 100644 index 50edc1292e..0000000000 --- a/src/packages/project/sync/server.ts +++ /dev/null @@ -1,567 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -/* -SyncTable server channel -- used for supporting realtime sync -between project and browser client. - -TODO: - -- [ ] If initial query fails, need to raise exception. Right now it gets -silently swallowed in persistent mode... -*/ - -// How long to wait from when we hit 0 clients until closing this channel. -// Making this short saves memory and cpu. -// Making it longer reduces the potential time to open a file, e.g., if you -// disconnect then reconnect, e.g., by refreshing your browser. -// Related to https://github.com/sagemathinc/cocalc/issues/5627 -// and https://github.com/sagemathinc/cocalc/issues/5823 -// and https://github.com/sagemathinc/cocalc/issues/5617 - -// This is a hard upper bound on the number of browser sessions that could -// have the same file open at once. We put some limit on it, to at least -// limit problems from bugs which crash projects (since each connection uses -// memory, and it adds up). Some customers want 100+ simultaneous users, -// so don't set this too low (except for dev)! -const MAX_CONNECTIONS = 500; - -// The frontend client code *should* prevent many connections, but some -// old broken clients may not work properly. This must be at least 2, -// since we can have two clients for a given channel at once if a file is -// being closed still, while it is reopened (e.g., when user does this: -// disconnect, change, close, open, reconnect). Also, this setting prevents -// some potentially malicious conduct, and also possible new clients with bugs. -// It is VERY important that this not be too small, since there is often -// a delay/timeout before a channel is properly closed. -const MAX_CONNECTIONS_FROM_ONE_CLIENT = 10; - -import { - synctable_no_changefeed, - synctable_no_database, - SyncTable, - VersionedChange, - set_debug, -} from "@cocalc/sync/table"; - -// Only uncomment this for an intense level of debugging. -// set_debug(true); -// @ts-ignore -- typescript nonsense. -const _ = set_debug; - -import { init_syncdoc, getSyncDocFromSyncTable } from "./sync-doc"; -import { key, register_synctable } from "./open-synctables"; -import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; -import { once } from "@cocalc/util/async-utils"; -import { delay } from "awaiting"; -import { close, deep_copy, len } from "@cocalc/util/misc"; -import { registerListingsTable } from "./listings"; -import { register_project_info_table } from "./project-info"; -import { register_project_status_table } from "./project-status"; -import { register_usage_info_table } from "./usage-info"; -import Client from "@cocalc/sync-client"; -import { getJupyterRedux } from "@cocalc/jupyter/kernel"; -import { JUPYTER_SYNCDB_EXTENSIONS } from "@cocalc/util/jupyter/names"; - -type Query = { [key: string]: any }; - -interface Spark { - address: { ip: string }; - id: string; - conn: { - id: string; - write: (obj: any) => boolean; - once: (str: string, fn: Function) => void; - on: (str: string, fn: Function) => void; - writable: boolean; - }; - write: (obj: any) => boolean; - end: (...args) => void; - on: (str: string, fn: Function) => void; -} - -interface Channel { - write: (obj: any) => boolean; - on: (str: string, fn: Function) => void; - forEach: (fn: Function) => void; - destroy: Function; -} - -interface Primus { - channel: (str: string) => Channel; -} - -interface Logger { - debug: Function; -} - -import stringify from "json-stable-stringify"; -import { sha1 } from "@cocalc/backend/sha1"; - -const COCALC_EPHEMERAL_STATE: boolean = - process.env.COCALC_EPHEMERAL_STATE === "yes"; - -class SyncTableChannel { - private synctable: SyncTable; - private client: Client; - private logger: Logger; - public readonly name: string; - private query: Query; - private options: any[] = []; - private query_string: string; - private channel: Channel; - private closed: boolean = false; - private closing: boolean = false; - private num_connections: { n: number; changed: Date } = { - n: 0, - changed: new Date(), - }; - - // If true, do not use a database at all, even on the backend. - // Table is reset any time this object is created. This is - // useful, e.g., for tracking user cursor locations or other - // ephemeral state. - private ephemeral: boolean = false; - - // If true, do not close even if all clients have disconnected. - // This is used to keep sessions running, even when all browsers - // have closed, e.g., state for Sage worksheets, jupyter - // notebooks, etc., where user may want to close their browser - // (or just drop a connection temporarily) while a persistent stateful - // session continues running. - private persistent: boolean = false; - - private connections_from_one_client: { [id: string]: number } = {}; - - constructor({ - client, - primus, - query, - options, - logger, - name, - }: { - client: Client; - primus: Primus; - name: string; - query: Query; - options: any; - logger: Logger; - }) { - this.name = name; - this.client = client; - this.logger = logger; - this.query = query; - this.init_options(options); - if (COCALC_EPHEMERAL_STATE) { - // No matter what, we set ephemeral true when - // this env var is set, since all db access - // will be denied anyways. - this.ephemeral = true; - } - this.query_string = stringify(query)!; // used only for logging - this.channel = primus.channel(this.name); - this.log( - `creating new sync channel (persistent=${this.persistent}, ephemeral=${this.ephemeral})`, - ); - } - - public async init(): Promise { - this.init_handlers(); - await this.init_synctable(); - } - - private init_options(options): void { - if (options == null) { - return; - } - for (const option of deep_copy(options)) { - // deep_copy so do not mutate input options. - if (typeof option != "object" || option == null) { - throw Error("invalid options"); - } - for (const x of ["ephemeral", "persistent"]) { - // options that are only for project websocket tables. - if (option[x] != null) { - this[x] = option[x]; - delete option[x]; - } - } - if (len(option) > 0) { - // remaining synctable/database options. - this.options.push(option); - } - } - } - - private log(...args): void { - if (this.logger == null) return; - this.logger.debug( - `SyncTableChannel('${this.name}', '${this.query_string}'${ - this.closed ? ",CLOSED" : "" - }): `, - ...args, - ); - } - - private init_handlers(): void { - this.log("init_handlers"); - this.channel.on("connection", this.new_connection.bind(this)); - this.channel.on("disconnection", this.end_connection.bind(this)); - } - - private async init_synctable(): Promise { - this.log("init_synctable"); - let create_synctable: Function; - if (this.ephemeral) { - this.log("init_synctable -- ephemeral (no database)"); - create_synctable = synctable_no_database; - } else { - this.log("init_synctable -- persistent (but no changefeeds)"); - create_synctable = synctable_no_changefeed; - } - this.synctable = create_synctable(this.query, this.options, this.client); - - // if the synctable closes, then the channel should also close. - // I think this should happen, e.g., when we "close and halt" - // a jupyter notebook, which closes the synctable, triggering this. - this.synctable.once("closed", this.close.bind(this)); - - if (this.query[this.synctable.get_table()][0].string_id != null) { - register_synctable(this.query, this.synctable); - } - if (this.synctable.table === "syncstrings") { - this.log("init_synctable -- syncstrings: also initialize syncdoc..."); - init_syncdoc(this.client, this.synctable); - } - - this.synctable.on( - "versioned-changes", - this.send_versioned_changes_to_browsers.bind(this), - ); - - this.log("created synctable -- waiting for connected state"); - await once(this.synctable, "connected"); - this.log("created synctable -- now connected"); - - // broadcast synctable content to all connected clients. - this.broadcast_synctable_to_browsers(); - } - - private increment_connection_count(spark: Spark): number { - // account for new connection from this particular client. - let m: undefined | number = this.connections_from_one_client[spark.conn.id]; - if (m === undefined) m = 0; - return (this.connections_from_one_client[spark.conn.id] = m + 1); - } - - private decrement_connection_count(spark: Spark): number { - const m: undefined | number = - this.connections_from_one_client[spark.conn.id]; - if (m === undefined) { - return 0; - } - return (this.connections_from_one_client[spark.conn.id] = Math.max( - 0, - m - 1, - )); - } - - private async new_connection(spark: Spark): Promise { - // Now handle the connection - const n = this.num_connections.n + 1; - this.num_connections = { n, changed: new Date() }; - - // account for new connection from this particular client. - const m = this.increment_connection_count(spark); - - this.log( - `new connection from (address=${spark.address.ip}, conn=${spark.conn.id}) -- ${spark.id} -- num_connections = ${n} (from this client = ${m})`, - ); - - if (m > MAX_CONNECTIONS_FROM_ONE_CLIENT) { - const error = `Too many connections (${m} > ${MAX_CONNECTIONS_FROM_ONE_CLIENT}) from this client. You might need to refresh your browser.`; - this.log( - `${error} Waiting 15s, then killing new connection from ${spark.id}...`, - ); - await delay(15000); // minimize impact of client trying again, which it should do... - this.decrement_connection_count(spark); - spark.end({ error }); - return; - } - - if (n > MAX_CONNECTIONS) { - const error = `Too many connections (${n} > ${MAX_CONNECTIONS})`; - this.log( - `${error} Waiting 5s, then killing new connection from ${spark.id}`, - ); - await delay(5000); // minimize impact of client trying again, which it should do - this.decrement_connection_count(spark); - spark.end({ error }); - return; - } - - if (this.closed) { - this.log(`table closed: killing new connection from ${spark.id}`); - this.decrement_connection_count(spark); - spark.end(); - return; - } - if (this.synctable != null && this.synctable.get_state() == "closed") { - this.log(`table state closed: killing new connection from ${spark.id}`); - this.decrement_connection_count(spark); - spark.end(); - return; - } - if ( - this.synctable != null && - this.synctable.get_state() == "disconnected" - ) { - // Because synctable is being initialized for the first time, - // or it temporarily disconnected (e.g., lost hub), and is - // trying to reconnect. So just wait for it to connect. - await once(this.synctable, "connected"); - } - - // Now that table is connected, we can send initial mesg to browser - // with table state. - this.send_synctable_to_browser(spark); - - spark.on("data", async (mesg) => { - try { - await this.handle_mesg_from_browser(mesg); - } catch (err) { - spark.write({ error: `error handling mesg -- ${err}` }); - this.log("error handling mesg -- ", err, err.stack); - } - }); - } - - private async end_connection(spark: Spark): Promise { - // This should never go below 0 (that would be a bug), but let's - // just ewnsure it doesn't since if it did that would weirdly break - // things for users as the table would keep trying to close. - const n = Math.max(0, this.num_connections.n - 1); - this.num_connections = { n, changed: new Date() }; - - const m = this.decrement_connection_count(spark); - this.log( - `spark event -- end connection ${spark.address.ip} -- ${spark.id} -- num_connections = ${n} (from this client = ${m})`, - ); - - this.check_if_should_save_or_close(); - } - - private send_synctable_to_browser(spark: Spark): void { - if (this.closed || this.closing || this.synctable == null) return; - this.log("send_synctable_to_browser"); - spark.write({ init: this.synctable.initial_version_for_browser_client() }); - } - - private broadcast_synctable_to_browsers(): void { - if (this.closed || this.closing || this.synctable == null) return; - this.log("broadcast_synctable_to_browsers"); - const x = { init: this.synctable.initial_version_for_browser_client() }; - this.channel.write(x); - } - - /* This is called when a user disconnects. This always triggers a save to - disk. It may also trigger closing the file in some cases. */ - private async check_if_should_save_or_close() { - if (this.closed) { - // don't bother if either already closed - return; - } - this.log("check_if_should_save_or_close: save to disk if possible"); - try { - await this.save_if_possible(); - } catch (err) { - // the name "save if possible" suggests this should be non-fatal. - this.log( - "check_if_should_save_or_close: WARNING: unable to save -- ", - err, - ); - } - const { n } = this.num_connections ?? {}; - this.log("check_if_should_save_or_close", { n }); - if (!this.persistent && n === 0) { - this.log("check_if_should_save_or_close: close if possible"); - await this.close_if_possible(); - } - } - - private handle_mesg_from_browser = async (mesg: any): Promise => { - // do not log the actual mesg, since it can be huge and make the logfile dozens of MB. - // Temporarily enable as needed for debugging purposes. - //this.log("handle_mesg_from_browser ", { mesg }); - if (this.closed) { - throw Error("received mesg from browser AFTER close"); - } - if (mesg == null) { - throw Error("mesg must not be null"); - } - if (mesg.timed_changes != null) { - this.synctable.apply_changes_from_browser_client(mesg.timed_changes); - await this.synctable.save(); - } - }; - - private send_versioned_changes_to_browsers = ( - versioned_changes: VersionedChange[], - ): void => { - if (this.closed) return; - this.log("send_versioned_changes_to_browsers"); - const x = { versioned_changes }; - this.channel.write(x); - }; - - private async save_if_possible(): Promise { - if (this.closed || this.closing) { - return; // closing or already closed - } - this.log("save_if_possible: saves changes to database"); - await this.synctable.save(); - if (this.synctable.table === "syncstrings") { - this.log("save_if_possible: also fetch syncdoc"); - const syncdoc = getSyncDocFromSyncTable(this.synctable); - if (syncdoc != null) { - const path = syncdoc.get_path(); - this.log("save_if_possible: saving syncdoc to disk", { path }); - if (path.endsWith("." + JUPYTER_SYNCDB_EXTENSIONS)) { - // treat jupyter notebooks in a special way, since they have - // an aux .ipynb file that the syncdoc doesn't know about. In - // this case we save the ipynb to disk, not just the hidden - // syncdb file. - const { actions } = await getJupyterRedux(syncdoc); - if (actions == null) { - this.log("save_if_possible: jupyter -- actions is null"); - } else { - this.log("save_if_possible: jupyter -- saving to ipynb"); - await actions.save_ipynb_file(); - } - } - await syncdoc.save_to_disk(); - } else { - this.log("save_if_possible: no syncdoc"); - } - } - } - - private async close_if_possible(): Promise { - if (this.closed || this.closing) { - return; // closing or already closed - } - const { n, changed } = this.num_connections; - const delay = Date.now() - changed.valueOf(); - this.log( - `close_if_possible: there are ${n} connections and delay=${delay}`, - ); - if (n === 0) { - this.log(`close_if_possible: close this SyncTableChannel atomically`); - // actually close - this.close(); - } else { - this.log(`close_if_possible: NOT closing this SyncTableChannel`); - } - } - - private close(): void { - if (this.closed) { - return; - } - this.log("close: closing"); - this.closing = true; - delete synctable_channels[this.name]; - this.channel.destroy(); - this.synctable.close_no_async(); - this.log("close: closed"); - close(this); // don't call this.log after this! - this.closed = true; - } - - public get_synctable(): SyncTable { - return this.synctable; - } -} - -const synctable_channels: { [name: string]: SyncTableChannel } = {}; - -function createKey(args): string { - return stringify([args[3], args[4]])!; -} - -function channel_name(query: any, options: any[]): string { - // stable identifier to this query + options across - // project restart, etc. We first make the options - // as canonical as we can: - const opts = {}; - for (const x of options) { - for (const key in x) { - opts[key] = x[key]; - } - } - // It's critical that we dedup the synctables having - // to do with sync-doc's. A problem case is multiple - // queries for the same table, due to the time cutoff - // for patches after making a snapshot. - let q: string; - try { - q = key(query); - } catch { - // throws an error if the table doesn't have a string_id; - // that's fine - in this case, just make a key out of the query. - q = query; - } - const y = stringify([q, opts])!; - const s = sha1(y); - return `sync:${s}`; -} - -async function synctable_channel0( - client: any, - primus: any, - logger: any, - query: any, - options: any[], -): Promise { - const name = channel_name(query, options); - logger.debug("synctable_channel", JSON.stringify(query), name); - if (synctable_channels[name] === undefined) { - synctable_channels[name] = new SyncTableChannel({ - client, - primus, - name, - query, - options, - logger, - }); - await synctable_channels[name].init(); - if (query?.listings != null) { - registerListingsTable(synctable_channels[name].get_synctable(), query); - } else if (query?.project_info != null) { - register_project_info_table( - synctable_channels[name].get_synctable(), - logger, - client.client_id(), - ); - } else if (query?.project_status != null) { - register_project_status_table( - synctable_channels[name].get_synctable(), - logger, - client.client_id(), - ); - } else if (query?.usage_info != null) { - register_usage_info_table( - synctable_channels[name].get_synctable(), - client.client_id(), - ); - } - } - return name; -} - -export const synctable_channel = reuseInFlight(synctable_channel0, { - createKey, -}); diff --git a/src/packages/project/sync/sync-doc.ts b/src/packages/project/sync/sync-doc.ts deleted file mode 100644 index a8000e743d..0000000000 --- a/src/packages/project/sync/sync-doc.ts +++ /dev/null @@ -1,310 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -/* -Backend project support for using syncdocs. - -This is mainly responsible for: - -- loading and saving files to disk -- executing code - -*/ - -import { SyncTable } from "@cocalc/sync/table"; -import { SyncDB } from "@cocalc/sync/editor/db/sync"; -import { SyncString } from "@cocalc/sync/editor/string/sync"; -import type Client from "@cocalc/sync-client"; -import { once } from "@cocalc/util/async-utils"; -import { filename_extension, original_path } from "@cocalc/util/misc"; -import { initJupyterRedux, removeJupyterRedux } from "@cocalc/jupyter/kernel"; -import { EventEmitter } from "events"; -import { COMPUTER_SERVER_DB_NAME } from "@cocalc/util/compute/manager"; -import computeServerOpenFileTracking from "./compute-server-open-file-tracking"; -import { getLogger } from "@cocalc/backend/logger"; -import { JUPYTER_SYNCDB_EXTENSIONS } from "@cocalc/util/jupyter/names"; - -const logger = getLogger("project:sync:sync-doc"); - -type SyncDoc = SyncDB | SyncString; - -const COCALC_EPHEMERAL_STATE: boolean = - process.env.COCALC_EPHEMERAL_STATE === "yes"; - -export class SyncDocs extends EventEmitter { - private syncdocs: { [path: string]: SyncDoc } = {}; - private closing: Set = new Set(); - - async close(path: string): Promise { - const doc = this.get(path); - if (doc == null) { - logger.debug(`SyncDocs: close ${path} -- no need, as it is not opened`); - return; - } - try { - logger.debug(`SyncDocs: close ${path} -- starting close`); - this.closing.add(path); - // As soon as this close starts, doc is in an undefined state. - // Also, this can take an **unbounded** amount of time to finish, - // since it tries to save the patches table (among other things) - // to the database, and if there is no connection from the hub - // to this project, then it will simply wait however long it takes - // until we get a connection (and there is no timeout). That is - // perfectly fine! E.g., a user closes their browser connected - // to a project, then comes back 8 hours later and tries to open - // this document when they resume their browser. During those entire - // 8 hours, the project might have been waiting to reconnect, just - // so it could send the patches from patches_list to the database. - // It does that, then finishes this async doc.close(), releases - // the lock, and finally the user gets to open their file. See - // https://github.com/sagemathinc/cocalc/issues/5823 for how not being - // careful with locking like this resulted in a very difficult to - // track down heisenbug. See also - // https://github.com/sagemathinc/cocalc/issues/5617 - await doc.close(); - logger.debug(`SyncDocs: close ${path} -- successfully closed`); - } finally { - // No matter what happens above when it finishes, we clear it - // and consider it closed. - // There is perhaps a chance closing fails above (no idea how), - // but we don't want it to be impossible to attempt to open - // the path again I.e., we don't want to leave around a lock. - logger.debug(`SyncDocs: close ${path} -- recording that close succeeded`); - delete this.syncdocs[path]; - this.closing.delete(path); - // I think close-${path} is used only internally in this.create below - this.emit(`close-${path}`); - // This is used by computeServerOpenFileTracking - this.emit("close", path); - } - } - - get(path: string): SyncDoc | undefined { - return this.syncdocs[path]; - } - - getOpenPaths = (): string[] => { - return Object.keys(this.syncdocs); - }; - - isOpen = (path: string): boolean => { - return this.syncdocs[path] != null; - }; - - async create(type, opts): Promise { - const path = opts.path; - if (this.closing.has(path)) { - logger.debug( - `SyncDocs: create ${path} -- waiting for previous version to completely finish closing...`, - ); - await once(this, `close-${path}`); - logger.debug(`SyncDocs: create ${path} -- successfully closed.`); - } - let doc; - switch (type) { - case "string": - doc = new SyncString(opts); - break; - case "db": - doc = new SyncDB(opts); - break; - default: - throw Error(`unknown syncdoc type ${type}`); - } - this.syncdocs[path] = doc; - logger.debug(`SyncDocs: create ${path} -- successfully created`); - // This is used by computeServerOpenFileTracking: - this.emit("open", path); - if (path == COMPUTER_SERVER_DB_NAME) { - logger.debug( - "SyncDocs: also initializing open file tracking for ", - COMPUTER_SERVER_DB_NAME, - ); - computeServerOpenFileTracking(this, doc); - } - return doc; - } - - async closeAll(filename: string): Promise { - logger.debug(`SyncDocs: closeAll("${filename}")`); - for (const path in this.syncdocs) { - if (path == filename || path.startsWith(filename + "/")) { - await this.close(path); - } - } - } -} - -const syncDocs = new SyncDocs(); - -// The "synctable" here is EXACTLY ONE ENTRY of the syncstrings table. -// That is the table in the postgresql database that tracks the path, -// save state, document type, etc., of a syncdoc. It's called syncstrings -// instead of syncdoc_metadata (say) because it was created when we only -// used strings for sync. - -export function init_syncdoc(client: Client, synctable: SyncTable): void { - if (synctable.get_table() !== "syncstrings") { - throw Error("table must be 'syncstrings'"); - } - if (synctable.get_state() == "closed") { - throw Error("synctable must not be closed"); - } - // It's the right type of table and not closed. Now do - // the real setup work (without blocking). - init_syncdoc_async(client, synctable); -} - -// If there is an already existing syncdoc for this path, -// return it; otherwise, return undefined. This is useful -// for getting a reference to a syncdoc, e.g., for prettier. -export function get_syncdoc(path: string): SyncDoc | undefined { - return syncDocs.get(path); -} - -export function getSyncDocFromSyncTable(synctable: SyncTable) { - const { opts } = get_type_and_opts(synctable); - return get_syncdoc(opts.path); -} - -async function init_syncdoc_async( - client: Client, - synctable: SyncTable, -): Promise { - function log(...args): void { - logger.debug("init_syncdoc_async: ", ...args); - } - - log("waiting until synctable is ready"); - await wait_until_synctable_ready(synctable); - log("synctable ready. Now getting type and opts"); - const { type, opts } = get_type_and_opts(synctable); - const project_id = (opts.project_id = client.client_id()); - // log("type = ", type); - // log("opts = ", JSON.stringify(opts)); - opts.client = client; - log(`now creating syncdoc ${opts.path}...`); - let syncdoc; - try { - syncdoc = await syncDocs.create(type, opts); - } catch (err) { - log(`ERROR creating syncdoc -- ${err.toString()}`, err.stack); - // TODO: how to properly inform clients and deal with this?! - return; - } - synctable.on("closed", () => { - log("synctable closed, so closing syncdoc", opts.path); - syncDocs.close(opts.path); - }); - - syncdoc.on("error", (err) => { - log(`syncdoc error -- ${err}`); - syncDocs.close(opts.path); - }); - - // Extra backend support in some cases, e.g., Jupyter, Sage, etc. - const ext = filename_extension(opts.path); - log("ext = ", ext); - switch (ext) { - case JUPYTER_SYNCDB_EXTENSIONS: - log("initializing Jupyter backend"); - await initJupyterRedux(syncdoc, client); - const path = original_path(syncdoc.get_path()); - synctable.on("closed", async () => { - log("removing Jupyter backend"); - await removeJupyterRedux(path, project_id); - }); - break; - } -} - -async function wait_until_synctable_ready(synctable: SyncTable): Promise { - if (synctable.get_state() == "disconnected") { - logger.debug("wait_until_synctable_ready: wait for synctable be connected"); - await once(synctable, "connected"); - } - - const t = synctable.get_one(); - if (t != null) { - logger.debug("wait_until_synctable_ready: currently", t.toJS()); - } - logger.debug( - "wait_until_synctable_ready: wait for document info to get loaded into synctable...", - ); - // Next wait until there's a document in the synctable, since that will - // have the path, patch type, etc. in it. That is set by the frontend. - function is_ready(): boolean { - const t = synctable.get_one(); - if (t == null) { - logger.debug("wait_until_synctable_ready: is_ready: table is null still"); - return false; - } else { - logger.debug("wait_until_synctable_ready: is_ready", JSON.stringify(t)); - return t.has("path"); - } - } - await synctable.wait(is_ready, 0); - logger.debug("wait_until_synctable_ready: document info is now in synctable"); -} - -function get_type_and_opts(synctable: SyncTable): { type: string; opts: any } { - const s = synctable.get_one(); - if (s == null) { - throw Error("synctable must not be empty"); - } - const path = s.get("path"); - if (typeof path != "string") { - throw Error("path must be a string"); - } - const opts = { path, ephemeral: COCALC_EPHEMERAL_STATE }; - let type: string = ""; - - let doctype = s.get("doctype"); - if (doctype != null) { - try { - doctype = JSON.parse(doctype); - } catch { - doctype = {}; - } - if (doctype.opts != null) { - for (const k in doctype.opts) { - opts[k] = doctype.opts[k]; - } - } - type = doctype.type; - } - if (type !== "db" && type !== "string") { - // fallback type - type = "string"; - } - return { type, opts }; -} - -export async function syncdoc_call(path: string, mesg: any): Promise { - logger.debug("syncdoc_call", path, mesg); - const doc = syncDocs.get(path); - if (doc == null) { - logger.debug("syncdoc_call -- not open: ", path); - return "not open"; - } - switch (mesg.cmd) { - case "close": - logger.debug("syncdoc_call -- now closing: ", path); - await syncDocs.close(path); - logger.debug("syncdoc_call -- closed: ", path); - return "successfully closed"; - default: - throw Error(`unknown command ${mesg.cmd}`); - } -} - -// This is used when deleting a file/directory -// filename may be a directory or actual filename -export async function close_all_syncdocs_in_tree( - filename: string, -): Promise { - logger.debug("close_all_syncdocs_in_tree", filename); - return await syncDocs.closeAll(filename); -} diff --git a/src/packages/project/sync/usage-info.ts b/src/packages/project/sync/usage-info.ts deleted file mode 100644 index ad3cac0412..0000000000 --- a/src/packages/project/sync/usage-info.ts +++ /dev/null @@ -1,183 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -// usage info for a specific file path, derived from the more general project info, -// which includes all processes and other stats - -import { SyncTable, SyncTableState } from "@cocalc/sync/table"; -import { once } from "@cocalc/util/async-utils"; -import { close, merge } from "@cocalc/util/misc"; -import { UsageInfoServer } from "../usage-info"; -import type { ImmutableUsageInfo, UsageInfo } from "@cocalc/util/types/project-usage-info"; -import { getLogger } from "@cocalc/backend/logger"; - -const L = getLogger("sync:usage-info"); - -class UsageInfoTable { - private readonly table?: SyncTable; // might be removed by close() - private readonly project_id: string; - private readonly servers: { [path: string]: UsageInfoServer } = {}; - private readonly log: Function; - - constructor(table: SyncTable, project_id: string) { - this.project_id = project_id; - this.log = L.extend("table").debug; - this.table = table; - this.setup_watchers(); - } - - public close(): void { - this.log("close"); - for (const path in this.servers) { - this.stop_server(path); - } - close(this); - } - - // Start watching any paths that have recent interest (so this is not - // in response to a *change* after starting). - private async setup_watchers(): Promise { - if (this.table == null) return; // closed - if (this.table.get_state() == ("init" as SyncTableState)) { - await once(this.table, "state"); - } - if (this.table.get_state() != ("connected" as SyncTableState)) { - return; // game over - } - this.table.get()?.forEach((val) => { - const path = val.get("path"); - if (path == null) return; - if (this.servers[path] == null) return; // already watching - }); - this.log("setting up 'on.change'"); - this.table.on("change", this.handle_change_event.bind(this)); - } - - private async remove_stale_servers(): Promise { - if (this.table == null) return; // closed - if (this.table.get_state() != ("connected" as SyncTableState)) return; - const paths: string[] = []; - this.table.get()?.forEach((val) => { - const path = val.get("path"); - if (path == null) return; - paths.push(path); - }); - for (const path of Object.keys(this.servers)) { - if (!paths.includes(path)) { - this.stop_server(path); - } - } - } - - private is_ready(): boolean { - return !!this.table?.is_ready(); - } - - private get_table(): SyncTable { - if (!this.is_ready() || this.table == null) { - throw Error("table not ready"); - } - return this.table; - } - - async set(obj: { path: string; usage?: UsageInfo }): Promise { - this.get_table().set( - merge({ project_id: this.project_id }, obj), - "shallow" - ); - await this.get_table().save(); - } - - public get(path: string): ImmutableUsageInfo | undefined { - const x = this.get_table().get(JSON.stringify([this.project_id, path])); - if (x == null) return x; - return x as unknown as ImmutableUsageInfo; - // NOTE: That we have to use JSON.stringify above is an ugly shortcoming - // of the get method in @cocalc/sync/table/synctable.ts - // that could probably be relatively easily fixed. - } - - private handle_change_event(keys: string[]): void { - // this.log("handle_change_event", JSON.stringify(keys)); - for (const key of keys) { - this.handle_change(JSON.parse(key)[1]); - } - this.remove_stale_servers(); - } - - private handle_change(path: string): void { - this.log("handle_change", path); - const cur = this.get(path); - if (cur == null) return; - // Make sure we watch this path for updates, since there is genuine current interest. - this.ensure_watching(path); - this.set({ path }); - } - - private ensure_watching(path: string): void { - if (this.servers[path] != null) { - // We are already watching this path, so nothing more to do. - return; - } - - try { - this.start_watching(path); - } catch (err) { - this.log("failed to start watching", err); - } - } - - private start_watching(path: string): void { - this.log(`start_watching ${path}`); - if (this.servers[path] != null) return; - const server = new UsageInfoServer(path); - - server.on("usage", (usage: UsageInfo) => { - // this.log(`watching/usage:`, usage); - try { - if (!this.is_ready()) return; - this.set({ path, usage }); - } catch (err) { - this.log(`compute_listing("${path}") error: "${err}"`); - } - }); - - server.start(); - - this.servers[path] = server; - } - - private stop_server(path: string): void { - const s = this.servers[path]; - if (s == null) return; - delete this.servers[path]; - s.stop(); - this.remove_path(path); - } - - private async remove_path(path: string): Promise { - if (!this.is_ready()) return; - this.log("remove_path", path); - await this.get_table().delete({ project_id: this.project_id, path }); - } -} - -let usage_info_table: UsageInfoTable | undefined = undefined; -export function register_usage_info_table( - table: SyncTable, - project_id: string -): void { - L.debug("register_usage_info_table"); - if (usage_info_table != null) { - // There was one sitting around wasting space so clean it up - // before making a new one. - usage_info_table.close(); - } - usage_info_table = new UsageInfoTable(table, project_id); -} - -export function get_usage_info_table(): UsageInfoTable | undefined { - return usage_info_table; -} diff --git a/src/packages/project/tsconfig.json b/src/packages/project/tsconfig.json index 3208f0532a..2a2b4f210f 100644 --- a/src/packages/project/tsconfig.json +++ b/src/packages/project/tsconfig.json @@ -12,11 +12,10 @@ { "path": "../backend" }, { "path": "../comm" }, { "path": "../jupyter" }, - { "path": "../nats" }, + { "path": "../conat" }, { "path": "../sync" }, { "path": "../sync-client" }, { "path": "../sync-fs" }, - { "path": "../terminal" }, { "path": "../util" } ] } diff --git a/src/packages/project/usage-info/server.ts b/src/packages/project/usage-info.ts similarity index 52% rename from src/packages/project/usage-info/server.ts rename to src/packages/project/usage-info.ts index 9f0c4ff9bb..4f99ff34ba 100644 --- a/src/packages/project/usage-info/server.ts +++ b/src/packages/project/usage-info.ts @@ -8,21 +8,28 @@ Usage Info Server This derives usage information (cpu, mem, etc.) for a specific "path" (e.g. the corresponding jupyter process for a notebook) -from the ProjectInfoServer (which collects data about everything) +from the ProjectInfoServer (which collects data about everything). + +It is made available via a service in @cocalc/conat/project/usage-info. */ -import { delay } from "awaiting"; import { EventEmitter } from "node:events"; - -import { getLogger } from "../logger"; -import { ProjectInfoServer, get_ProjectInfoServer } from "../project-info"; +import { getLogger } from "@cocalc/project/logger"; +import { + ProjectInfoServer, + get_ProjectInfoServer, +} from "@cocalc/project/project-info"; import { Process, ProjectInfo } from "@cocalc/util/types/project-info/types"; import type { UsageInfo } from "@cocalc/util/types/project-usage-info"; -import { throttle } from "lodash"; +import { + createUsageInfoService, + type UsageInfoService, +} from "@cocalc/conat/project/usage-info"; +import { compute_server_id, project_id } from "@cocalc/project/data"; -const L = getLogger("usage-info:server").debug; +export const UPDATE_INTERVAL_S = 2; -const throttled_dbg = throttle((...args) => L(...args), 10000); +const logger = getLogger("usage-info"); function is_diff(prev: UsageInfo, next: UsageInfo, key: keyof UsageInfo) { // we assume a,b >= 0, hence we leave out Math.abs operations @@ -32,77 +39,100 @@ function is_diff(prev: UsageInfo, next: UsageInfo, key: keyof UsageInfo) { return Math.abs(b - a) / Math.min(a, b) > 0.05; } +let server: UsageInfoService | null = null; +export function init() { + server = createUsageInfoService({ + project_id, + compute_server_id, + createUsageInfoServer: (path) => new UsageInfoServer(path), + }); +} + +export function close() { + server?.close(); + server = null; +} + export class UsageInfoServer extends EventEmitter { private readonly dbg: Function; - private running = false; - private readonly testing: boolean; private readonly project_info: ProjectInfoServer; private readonly path: string; private info?: ProjectInfo; private usage?: UsageInfo; private last?: UsageInfo; - constructor(path, testing = false) { + constructor(path: string) { super(); - this.testing = testing; this.path = path; - this.dbg = L; + this.dbg = (...args) => logger.debug(this.path, ...args); this.project_info = get_ProjectInfoServer(); - this.dbg("starting"); + this.project_info.on("info", this.handleUpdate); } - private async init(): Promise { - this.project_info.start(); - this.project_info.on("info", (info) => { - //this.dbg(`got info timestamp=${info.timestamp}`); - this.info = info; - this.update(); - }); - } + close = (): void => { + this.project_info?.removeListener("info", this.handleUpdate); + // @ts-ignore + delete this.project_info; + // @ts-ignore + delete this.dbg; + // @ts-ignore + delete this.path; + }; + + private handleUpdate = (info) => { + this.info = info; + this.update(); + }; // get the process at the given path – for now, that only works for jupyter notebooks - private path_process(): Process | undefined { - if (this.info?.processes == null) return; + private getProcessAtPath = (): Process | undefined => { + if (this.info?.processes == null) { + return; + } for (const p of Object.values(this.info.processes)) { const cocalc = p.cocalc; - if (cocalc == null || cocalc.type != "jupyter") continue; - if (cocalc.path == this.path) return p; + if (cocalc == null || cocalc.type != "jupyter") { + continue; + } + if (cocalc.path == this.path) { + return p; + } } - } + }; // we compute the total cpu and memory usage sum for the given PID // this is a quick recursive traverse, with "stats" as the accumulator - private proces_tree_stats(ppid: number, stats) { + private processTreeStats = (ppid: number, stats) => { const procs = this.info?.processes; if (procs == null) return; for (const proc of Object.values(procs)) { if (proc.ppid != ppid) continue; - this.proces_tree_stats(proc.pid, stats); + this.processTreeStats(proc.pid, stats); stats.mem += proc.stat.mem.rss; stats.cpu += proc.cpu.pct; } - } + }; // cpu usage sum of all children - private usage_children(pid): { cpu: number; mem: number } { + private cpuUsageSumOfChildren = (pid): { cpu: number; mem: number } => { const stats = { mem: 0, cpu: 0 }; - this.proces_tree_stats(pid, stats); + this.processTreeStats(pid, stats); return stats; - } + }; // we silently treat non-existing information as zero usage - private path_usage_info(): { + private pathUsageInfo = (): { cpu: number; cpu_chld: number; mem: number; mem_chld: number; - } { - const proc = this.path_process(); + } => { + const proc = this.getProcessAtPath(); if (proc == null) { return { cpu: 0, mem: 0, cpu_chld: 0, mem_chld: 0 }; } else { // we send whole numbers. saves bandwidth and won't be displayed anyways - const children = this.usage_children(proc.pid); + const children = this.cpuUsageSumOfChildren(proc.pid); return { cpu: Math.round(proc.cpu.pct), cpu_chld: Math.round(children.cpu), @@ -110,106 +140,62 @@ export class UsageInfoServer extends EventEmitter { mem_chld: Math.round(children.mem), }; } - } + }; // this function takes the "info" we have (+ more maybe?) // and derives specific information for the notebook (future: also other file types) // at the given path. - private update(): void { + private update = (): void => { if (this.info == null) { - L("was told to update, but there is no ProjectInfo"); + this.dbg("no info"); return; } const cg = this.info.cgroup; const du = this.info.disk_usage; - if (cg == null || du == null) { - // I'm seeing situations where I get many of these a second, - // and that isn't useful, hence throttling. - throttled_dbg("info incomplete, can't send usage data", this.path); - return; - } - const mem_rss = cg.mem_stat.total_rss + (du.tmp?.usage ?? 0); - const mem_tot = cg.mem_stat.hierarchical_memory_limit; + const mem_rss = (cg?.mem_stat?.total_rss ?? 0) + (du?.tmp?.usage ?? 0); + const mem_tot = cg?.mem_stat?.hierarchical_memory_limit ?? 0; const usage = { time: Date.now(), - ...this.path_usage_info(), + ...this.pathUsageInfo(), mem_limit: mem_tot, - cpu_limit: cg.cpu_cores_limit, + cpu_limit: cg?.cpu_cores_limit ?? 0, mem_free: Math.max(0, mem_tot - mem_rss), }; // this.dbg("usage", usage); - if (this.should_update(usage)) { + if (this.shouldUpdate(usage)) { this.usage = usage; this.emit("usage", this.usage); this.last = this.usage; } - } + }; // only cause to emit a change if it changed significantly (more than x%), // or if it changes close to zero (in particular, if cpu usage is low again) - private should_update(usage: UsageInfo): boolean { - if (this.last == null) return true; - if (usage == null) return false; + private shouldUpdate = (usage: UsageInfo): boolean => { + if (this.last == null) { + return true; + } + if (usage == null) { + return false; + } const keys: (keyof UsageInfo)[] = ["cpu", "mem", "cpu_chld", "mem_chld"]; for (const key of keys) { // we want everyone to know if essentially dropped to zero - if ((this.last[key] ?? 0) >= 1 && (usage[key] ?? 0) < 1) return true; + if ((this.last[key] ?? 0) >= 1 && (usage[key] ?? 0) < 1) { + return true; + } // … or of one of the values is significantly different - if (is_diff(usage, this.last, key)) return true; + if (is_diff(usage, this.last, key)) { + return true; + } } // … or if the remaining memory changed // i.e. if another process uses up a portion, there's less for the current notebook - if (is_diff(usage, this.last, "mem_free")) return true; - return false; - } - - private async get_usage(): Promise { - this.update(); - return this.usage; - } - - public stop(): void { - this.running = false; - } - - public async start(): Promise { - if (this.running) { - this.dbg("UsageInfoServer already running, cannot be started twice"); - } else { - await this._start(); - } - } - - private async _start(): Promise { - this.dbg("start"); - if (this.running) { - throw Error("Cannot start UsageInfoServer twice"); - } - this.running = true; - await this.init(); - - // emit once after startup - const usage = await this.get_usage(); - this.emit("usage", usage); - - while (this.testing) { - await delay(5000); - const usage = await this.get_usage(); - this.emit("usage", usage); + if (is_diff(usage, this.last, "mem_free")) { + return true; } - } -} - -// testing: $ ts-node server.ts -if (require.main === module) { - const uis = new UsageInfoServer("testing.ipynb", true); - uis.start(); - let cnt = 0; - uis.on("usage", (usage) => { - console.log(JSON.stringify(usage, null, 2)); - cnt += 1; - if (cnt >= 2) process.exit(); - }); + return false; + }; } diff --git a/src/packages/project/usage-info/const.ts b/src/packages/project/usage-info/const.ts deleted file mode 100644 index 850663ddca..0000000000 --- a/src/packages/project/usage-info/const.ts +++ /dev/null @@ -1,6 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -export const UPDATE_INTERVAL_S = 2; diff --git a/src/packages/project/usage-info/index.ts b/src/packages/project/usage-info/index.ts deleted file mode 100644 index 289bf34278..0000000000 --- a/src/packages/project/usage-info/index.ts +++ /dev/null @@ -1,6 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -export { UsageInfoServer } from "./server"; diff --git a/src/packages/server/accounts/is-admin.ts b/src/packages/server/accounts/is-admin.ts index 3524417dc0..bb0c7a5386 100644 --- a/src/packages/server/accounts/is-admin.ts +++ b/src/packages/server/accounts/is-admin.ts @@ -1,5 +1,14 @@ import userIsInGroup from "./is-in-group"; +import getPool from "@cocalc/database/pool"; export default async function isAdmin(account_id: string): Promise { return await userIsInGroup(account_id, "admin"); } + +export async function getAdmins(): Promise> { + const pool = getPool("long"); + const { rows } = await pool.query( + "SELECT account_id FROM accounts WHERE 'admin' = ANY(groups)", + ); + return new Set(rows.map((x) => x.account_id)); +} diff --git a/src/packages/server/accounts/send-email-verification.ts b/src/packages/server/accounts/send-email-verification.ts index f8f1058676..f9f726fd21 100644 --- a/src/packages/server/accounts/send-email-verification.ts +++ b/src/packages/server/accounts/send-email-verification.ts @@ -7,9 +7,13 @@ import { db } from "@cocalc/database"; import { verify_email_send_token } from "@cocalc/server/hub/auth"; import { callback2 as cb2 } from "@cocalc/util/async-utils"; import { isValidUUID } from "@cocalc/util/misc"; +import getLogger from "@cocalc/backend/logger"; + +const logger = getLogger("server:send-email-verification"); export default async function sendEmailVerification( account_id: string, + only_verify = true, ): Promise { if (!isValidUUID(account_id)) { throw Error("account_id is not valid"); @@ -17,10 +21,16 @@ export default async function sendEmailVerification( try { await cb2(verify_email_send_token, { account_id, - only_verify: true, + only_verify, database: db(), }); + logger.debug( + `successful sent verification email for account_id=${account_id}`, + ); } catch (err) { + logger.debug( + `failed to send verification email for account_id=${account_id} -- ${err}`, + ); return err.message; } return ""; diff --git a/src/packages/server/auth/sso/delete-passport.ts b/src/packages/server/auth/sso/delete-passport.ts index c614643602..7ab149f7e8 100644 --- a/src/packages/server/auth/sso/delete-passport.ts +++ b/src/packages/server/auth/sso/delete-passport.ts @@ -4,14 +4,13 @@ import type { PostgreSQL, } from "@cocalc/database/postgres/types"; import { _passport_key } from "@cocalc/database/postgres/passport"; +import { callback2 } from "@cocalc/util/async-utils"; export async function delete_passport( db: PostgreSQL, opts: DeletePassportOpts, ) { - db._dbg("delete_passport")( - JSON.stringify({ strategy: opts.strategy, id: opts.id }), - ); + db._dbg("delete_passport")(JSON.stringify({ strategy: opts.strategy })); if ( await isBlockedUnlinkStrategy({ @@ -19,16 +18,10 @@ export async function delete_passport( account_id: opts.account_id, }) ) { - const err_msg = `You are not allowed to unlink '${opts.strategy}'`; - if (typeof opts.cb === "function") { - opts.cb(err_msg); - return; - } else { - throw new Error(err_msg); - } + throw Error(`You are not allowed to unlink '${opts.strategy}'`); } - return db._query({ + await callback2(db._query, { query: "UPDATE accounts", jsonb_set: { // delete it @@ -37,6 +30,5 @@ export async function delete_passport( where: { "account_id = $::UUID": opts.account_id, }, - cb: opts.cb, }); } diff --git a/src/packages/server/compute/cloud/startup-script.ts b/src/packages/server/compute/cloud/startup-script.ts index 246457cca3..2573e2cd4f 100644 --- a/src/packages/server/compute/cloud/startup-script.ts +++ b/src/packages/server/compute/cloud/startup-script.ts @@ -18,6 +18,7 @@ import { CHECK_IN_PATH, CHECK_IN_PERIOD_S, } from "@cocalc/util/db-schema/compute-servers"; +import basePath from "@cocalc/backend/base-path"; // A one line startup script that grabs the latest version of the // real startup script via the API. This is important, e.g., if @@ -98,6 +99,9 @@ async function getApiServer() { if (!apiServer.includes("://")) { apiServer = `https://${apiServer}`; } + if (basePath.length > 1) { + apiServer += basePath; + } return apiServer; } @@ -154,6 +158,7 @@ set -v export COCALC_CLOUD=${cloud} export DEBIAN_FRONTEND=noninteractive export COCALC_LOCAL_SSD=${local_ssd ?? ""} +export CONAT_SERVER=${apiServer} ${defineSetStateFunction({ api_key, apiServer, compute_server_id })} diff --git a/src/packages/server/nats/api/db.ts b/src/packages/server/conat/api/db.ts similarity index 88% rename from src/packages/server/nats/api/db.ts rename to src/packages/server/conat/api/db.ts index d50ba5a85d..81b9258bd4 100644 --- a/src/packages/server/nats/api/db.ts +++ b/src/packages/server/conat/api/db.ts @@ -3,6 +3,7 @@ import userQuery from "@cocalc/database/user-query"; import { callback2 } from "@cocalc/util/async-utils"; import getPool from "@cocalc/database/pool"; import isCollaborator from "@cocalc/server/projects/is-collaborator"; +import { isValidUUID } from "@cocalc/util/misc"; export { userQuery }; export { fileUseTimes } from "./file-use-times"; @@ -79,3 +80,11 @@ export async function getLegacyTimeTravelPatches({ // we do NOT de-json this - leave it to the browser client to do that hard work... return blob.toString(); } + +export async function removeBlobTtls({ uuids }: { uuids: string[] }) { + const pool = getPool(); + const v = uuids.filter(isValidUUID); + if (v.length > 0) { + await pool.query("UPDATE blobs SET expire=NULL WHERE id::UUID=ANY($1::UUID[])", [v]); + } +} diff --git a/src/packages/server/nats/api/file-use-times.ts b/src/packages/server/conat/api/file-use-times.ts similarity index 92% rename from src/packages/server/nats/api/file-use-times.ts rename to src/packages/server/conat/api/file-use-times.ts index ba332096bd..22f7b46076 100644 --- a/src/packages/server/nats/api/file-use-times.ts +++ b/src/packages/server/conat/api/file-use-times.ts @@ -7,9 +7,9 @@ import getPool from "@cocalc/database/pool"; import type { FileUseTimesOptions, FileUseTimesResponse, -} from "@cocalc/nats/hub-api/db"; -import { dstream } from "@cocalc/nats/sync/dstream"; -import { patchesStreamName } from "@cocalc/nats/sync/synctable-stream"; +} from "@cocalc/conat/hub/api/db"; +import { dstream } from "@cocalc/conat/sync/dstream"; +import { patchesStreamName } from "@cocalc/conat/sync/synctable-stream"; export async function fileUseTimes({ project_id, diff --git a/src/packages/server/conat/api/index.ts b/src/packages/server/conat/api/index.ts new file mode 100644 index 0000000000..83a010ebac --- /dev/null +++ b/src/packages/server/conat/api/index.ts @@ -0,0 +1,189 @@ +/* +This is meant to be similar to the nexts pages http api/v2, but using NATS instead of HTTPS. + +To do development: + +1. Turn off conat-server handling for the hub by sending this message from a browser as an admin: + + await cc.client.conat_client.hub.system.terminate({service:'api'}) + +NOTE: there's no way to turn the auth back on in the hub, so you'll have to restart +your dev hub after doing the above. + +2. Run this script at the terminal: + + echo "require('@cocalc/server/conat/api').initAPI()" | COCALC_MODE='single-user' DEBUG_CONSOLE=yes DEBUG=cocalc:* node + + +3. Optional: start more servers -- requests get randomly routed to exactly one of them: + + echo "require('@cocalc/server/conat').default()" | COCALC_MODE='single-user' DEBUG_CONSOLE=yes DEBUG=cocalc:* node + echo "require('@cocalc/server/conat').default()" | COCALC_MODE='single-user' DEBUG_CONSOLE=yes DEBUG=cocalc:* node + + +To make use of this from a browser: + + await cc.client.conat_client.hub.system.getCustomize(['siteName']) + +or + + await cc.client.conat_client.callHub({name:"system.getCustomize", args:[['siteName']]}) + +When you make changes, just restart the above. All clients will instantly +use the new version after you restart, and there is no need to restart the hub +itself or any clients. + +To view requests in realtime + +cd packages/backend +pnpm conat-watch 'hub.*.*.api' --match-replies + +*/ + +import getLogger from "@cocalc/backend/logger"; +import { type HubApi, getUserId, transformArgs } from "@cocalc/conat/hub/api"; +import { conat } from "@cocalc/backend/conat"; +import userIsInGroup from "@cocalc/server/accounts/is-in-group"; +import { close as terminateChangefeedServer } from "@cocalc/database/conat/changefeed-api"; +import { close as terminatePersistServer } from "@cocalc/backend/conat/persist"; +import { delay } from "awaiting"; + +const logger = getLogger("server:conat:api"); + +export function initAPI() { + mainLoop(); +} + +let terminate = false; +async function mainLoop() { + let d = 3000; + let lastStart = 0; + while (!terminate) { + try { + lastStart = Date.now(); + await serve(); + } catch (err) { + logger.debug(`hub conat api service error -- ${err}`); + if (Date.now() - lastStart >= 30000) { + // it ran for a while, so no delay + logger.debug(`will restart immediately`); + d = 3000; + } else { + // it crashed quickly, so delay! + d = Math.min(20000, d * 1.25 + Math.random()); + logger.debug(`will restart in ${d}ms`); + await delay(d); + } + } + } +} + +async function serve() { + const subject = "hub.*.*.api"; + logger.debug(`initAPI -- subject='${subject}', options=`, { + queue: "0", + }); + const cn = await conat({ noCache: true }); + const api = await cn.subscribe(subject, { queue: "0" }); + for await (const mesg of api) { + (async () => { + try { + await handleMessage({ api, subject, mesg }); + } catch (err) { + logger.debug(`WARNING: unexpected error - ${err}`); + } + })(); + } +} + +async function handleMessage({ api, subject, mesg }) { + const request = mesg.data ?? ({} as any); + if (request.name == "system.terminate") { + // special hook so admin can terminate handling. This is useful for development. + const { account_id } = getUserId(mesg.subject); + if (!(!!account_id && (await userIsInGroup(account_id, "admin")))) { + mesg.respond({ error: "only admin can terminate" }); + return; + } + // TODO: could be part of handleApiRequest below, but done differently because + // one case halts this loop + const { service } = request.args[0] ?? {}; + logger.debug(`Terminate service '${service}'`); + if (service == "changefeeds") { + terminateChangefeedServer(); + mesg.respond({ status: "terminated", service }); + return; + } else if (service == "persist") { + terminatePersistServer(); + mesg.respond({ status: "terminated", service }); + return; + } else if (service == "api") { + // special hook so admin can terminate handling. This is useful for development. + console.warn("TERMINATING listening on ", subject); + logger.debug("TERMINATING listening on ", subject); + terminate = true; + mesg.respond({ status: "terminated", service }); + api.stop(); + return; + } else { + mesg.respond({ error: `Unknown service ${service}` }); + } + } else { + // we explicitly do NOT await this, since we want this hub server to handle + // potentially many messages at once, not one at a time! + handleApiRequest({ request, mesg }); + } +} + +async function handleApiRequest({ request, mesg }) { + let resp; + try { + const { account_id, project_id } = getUserId(mesg.subject); + const { name, args } = request as any; + logger.debug("handling hub.api request:", { + account_id, + project_id, + name, + }); + resp = (await getResponse({ name, args, account_id, project_id })) ?? null; + } catch (err) { + resp = { error: `${err}` }; + } + try { + await mesg.respond(resp); + } catch (err) { + // there's nothing we can do here, e.g., maybe NATS just died. + logger.debug( + `WARNING: error responding to hub.api request (client will receive no response) -- ${err}`, + ); + } +} + +import * as purchases from "./purchases"; +import * as db from "./db"; +import * as system from "./system"; +import * as projects from "./projects"; +import * as jupyter from "./jupyter"; + +export const hubApi: HubApi = { + system, + projects, + db, + purchases, + jupyter, +}; + +async function getResponse({ name, args, account_id, project_id }) { + const [group, functionName] = name.split("."); + const f = hubApi[group]?.[functionName]; + if (f == null) { + throw Error(`unknown function '${name}'`); + } + const args2 = await transformArgs({ + name, + args, + account_id, + project_id, + }); + return await f(...args2); +} diff --git a/src/packages/server/conat/api/jupyter.ts b/src/packages/server/conat/api/jupyter.ts new file mode 100644 index 0000000000..e27754663f --- /dev/null +++ b/src/packages/server/conat/api/jupyter.ts @@ -0,0 +1,3 @@ +export { execute } from "@cocalc/server/jupyter/execute"; +import kernels from "@cocalc/server/jupyter/kernels"; +export { kernels }; diff --git a/src/packages/server/nats/api/projects.ts b/src/packages/server/conat/api/projects.ts similarity index 58% rename from src/packages/server/nats/api/projects.ts rename to src/packages/server/conat/api/projects.ts index 7313246a5a..c2542b243a 100644 --- a/src/packages/server/nats/api/projects.ts +++ b/src/packages/server/conat/api/projects.ts @@ -1,10 +1,11 @@ import createProject from "@cocalc/server/projects/create"; export { createProject }; - import { type UserCopyOptions } from "@cocalc/util/db-schema/projects"; import { getProject } from "@cocalc/server/projects/control"; import isCollaborator from "@cocalc/server/projects/is-collaborator"; import { delay } from "awaiting"; +export * from "@cocalc/server/projects/collaborators"; +import isAdmin from "@cocalc/server/accounts/is-admin"; export async function copyPathBetweenProjects( opts: UserCopyOptions, @@ -13,6 +14,9 @@ export async function copyPathBetweenProjects( if (!account_id) { throw Error("user must be signed in"); } + if (opts.target_path == null) { + opts.target_path = opts.src_path; + } if (!(await isCollaborator({ account_id, project_id: src_project_id }))) { throw Error("user must be collaborator on source project"); } @@ -40,3 +44,34 @@ async function doCopyPathBetweenProjects(opts: UserCopyOptions) { await delay(opts.debug_delay_ms); } } + +import { callback2 } from "@cocalc/util/async-utils"; +import { db } from "@cocalc/database"; + +export async function setQuotas(opts: { + account_id: string; + project_id: string; + memory?: number; + memory_request?: number; + cpu_shares?: number; + cores?: number; + disk_quota?: number; + mintime?: number; + network?: number; + member_host?: number; + always_running?: number; +}): Promise { + if (!(await isAdmin(opts.account_id))) { + throw Error("Must be an admin to do admin search."); + } + const database = db(); + await callback2(database.set_project_settings, { + project_id: opts.project_id, + settings: opts, + }); + const project = await database.projectControl?.(opts.project_id); + // @ts-ignore + await project?.setAllQuotas(); +} + + diff --git a/src/packages/server/nats/api/purchases.ts b/src/packages/server/conat/api/purchases.ts similarity index 100% rename from src/packages/server/nats/api/purchases.ts rename to src/packages/server/conat/api/purchases.ts diff --git a/src/packages/server/conat/api/system.ts b/src/packages/server/conat/api/system.ts new file mode 100644 index 0000000000..c7c18ee487 --- /dev/null +++ b/src/packages/server/conat/api/system.ts @@ -0,0 +1,130 @@ +import getCustomize from "@cocalc/database/settings/customize"; +export { getCustomize }; +import { record_user_tracking } from "@cocalc/database/postgres/user-tracking"; +import { db } from "@cocalc/database"; +import manageApiKeys from "@cocalc/server/api/manage"; +export { manageApiKeys }; +import { type UserSearchResult } from "@cocalc/util/db-schema/accounts"; +import isAdmin from "@cocalc/server/accounts/is-admin"; +import search from "@cocalc/server/accounts/search"; +export { getNames } from "@cocalc/server/accounts/get-name"; +import { callback2 } from "@cocalc/util/async-utils"; + +export function ping() { + return { now: Date.now() }; +} + +export async function terminate() {} + +export async function userTracking({ + event, + value, + account_id, +}: { + event: string; + value: object; + account_id?: string; +}): Promise { + await record_user_tracking(db(), account_id!, event, value); +} + +export async function logClientError({ + account_id, + event, + error, +}: { + account_id?: string; + event: string; + error: string; +}): Promise { + await callback2(db().log_client_error, { + event, + error, + account_id, + }); +} + +export async function webappError(opts: object): Promise { + await callback2(db().webapp_error, opts); +} + +export { + generateUserAuthToken, + revokeUserAuthToken, +} from "@cocalc/server/auth/auth-token"; + +export async function userSearch({ + account_id, + query, + limit, + admin, + only_email, +}: { + account_id?: string; + query: string; + limit?: number; + admin?: boolean; + only_email?: boolean; +}): Promise { + if (!account_id) { + throw Error("You must be signed in to search for users."); + } + if (admin) { + if (!(await isAdmin(account_id))) { + throw Error("Must be an admin to do admin search."); + } + } else { + if (limit != null && limit > 50) { + // hard cap at 50... (for non-admin) + limit = 50; + } + } + return await search({ query, limit, admin, only_email }); +} + +import getEmailAddress from "@cocalc/server/accounts/get-email-address"; +import { createReset } from "@cocalc/server/auth/password-reset"; +export async function adminResetPasswordLink({ + account_id, + user_account_id, +}: { + account_id?: string; + user_account_id: string; +}): Promise { + if (!account_id || !(await isAdmin(account_id))) { + throw Error("must be an admin"); + } + const email = await getEmailAddress(user_account_id); + if (!email) { + throw Error("passwords are only defined for accounts with email"); + } + const id = await createReset(email, "", 60 * 60 * 24); // 24 hour ttl seems reasonable for this. + return `/auth/password-reset/${id}`; +} + +import sendEmailVerification0 from "@cocalc/server/accounts/send-email-verification"; + +export async function sendEmailVerification({ + account_id, + only_verify, +}: { + account_id?: string; + only_verify?: boolean; +}): Promise { + if (!account_id) { + throw Error("must be signed in"); + } + const resp = await sendEmailVerification0(account_id, only_verify); + if (resp) { + throw Error(resp); + } +} + +import { delete_passport } from "@cocalc/server/auth/sso/delete-passport"; +export async function deletePassport(opts: { + account_id: string; + strategy: string; + id: string; +}): Promise { + await delete_passport(db(), opts); +} diff --git a/src/packages/server/conat/configuration.ts b/src/packages/server/conat/configuration.ts new file mode 100644 index 0000000000..d7e3ab97b0 --- /dev/null +++ b/src/packages/server/conat/configuration.ts @@ -0,0 +1,32 @@ +/* +Load Conat configuration from the database, in case anything is set there. +*/ + +import getPool from "@cocalc/database/pool"; +import { + setConatServer, + setConatPassword, + setConatValkey, +} from "@cocalc/backend/data"; + +export async function loadConatConfiguration() { + const pool = getPool(); + const { rows } = await pool.query( + "SELECT name, value FROM server_settings WHERE name=ANY($1)", + [["conat_server", "conat_password", "conat_valkey"]], + ); + for (const { name, value } of rows) { + if (!value) { + continue; + } + if (name == "conat_password") { + setConatPassword(value.trim()); + } else if (name == "conat_server") { + setConatServer(value.trim()); + } else if (name == "conat_valkey") { + setConatValkey(value.trim()); + } else { + throw Error("bug"); + } + } +} diff --git a/src/packages/server/conat/index.ts b/src/packages/server/conat/index.ts new file mode 100644 index 0000000000..ad370b78c6 --- /dev/null +++ b/src/packages/server/conat/index.ts @@ -0,0 +1,47 @@ +import getLogger from "@cocalc/backend/logger"; +import { initAPI } from "./api"; +import { init as initChangefeedServer } from "@cocalc/database/conat/changefeed-api"; +import { init as initLLM } from "./llm"; +import { loadConatConfiguration } from "./configuration"; +import { createTimeService } from "@cocalc/conat/service/time"; +import { initPersistServer } from "@cocalc/backend/conat/persist"; +import { conatPersistCount, conatApiCount } from "@cocalc/backend/data"; + +export { loadConatConfiguration }; + +const logger = getLogger("server:conat"); + +export async function initConatChangefeedServer() { + logger.debug( + "initConatChangefeedServer: postgresql database query changefeeds", + ); + await loadConatConfiguration(); + initChangefeedServer(); +} + +export async function initConatPersist() { + logger.debug("initPersistServer: sqlite3 stream persistence", { + conatPersistCount, + }); + await loadConatConfiguration(); + for (let i = 0; i < conatPersistCount; i++) { + initPersistServer(); + } +} + +export async function initConatApi() { + logger.debug("initConatApi: the central api services", { conatApiCount }); + await loadConatConfiguration(); + + // do not block on any of these! + for (let i = 0; i < conatApiCount; i++) { + initAPI(); + } + initLLM(); + createTimeService(); +} + +export async function initConatCore() { + logger.debug("initConatApi: socketio websocsocket server on a port"); + await loadConatConfiguration(); +} diff --git a/src/packages/server/nats/llm.ts b/src/packages/server/conat/llm.ts similarity index 66% rename from src/packages/server/nats/llm.ts rename to src/packages/server/conat/llm.ts index 79a1d0c23c..6f06bdc169 100644 --- a/src/packages/server/nats/llm.ts +++ b/src/packages/server/conat/llm.ts @@ -1,4 +1,4 @@ -import { init as init0, close } from "@cocalc/nats/llm/server"; +import { init as init0, close } from "@cocalc/conat/llm/server"; import { evaluate } from "@cocalc/server/llm/index"; export async function init() { diff --git a/src/packages/server/conat/socketio/auth.test.ts b/src/packages/server/conat/socketio/auth.test.ts new file mode 100644 index 0000000000..bd09104a0a --- /dev/null +++ b/src/packages/server/conat/socketio/auth.test.ts @@ -0,0 +1,317 @@ +/* + +pnpm test `pwd`/auth.test.ts + +*/ + +import { getUser, isAllowed } from "./auth"; +import { inboxPrefix } from "@cocalc/conat/names"; + +// Mock the module where isCollaborator is exported from +jest.mock("@cocalc/server/projects/is-collaborator", () => ({ + __esModule: true, + default: jest.fn(), +})); + +import isCollaborator from "@cocalc/server/projects/is-collaborator"; + +const PUBSUB: ("pub" | "sub")[] = ["pub", "sub"]; + +const project_id = "00000000-0000-4000-8000-000000000000"; +const project_id2 = "00000000-0000-4000-8000-000000000001"; + +const account_id = "00000000-0000-4000-8000-000000000010"; +const account_id2 = "00000000-0000-4000-8000-000000000011"; + +describe("test isAllowed for non-authenticated", () => { + it("non-authenticated users can't do anything we try", async () => { + for (const subject of ["*", "public", "global", ">", "hub", "test"]) { + for (const type of PUBSUB) { + expect(await isAllowed({ user: null, type, subject })).toBe(false); + } + } + }); +}); + +describe("test isAllowed for hub", () => { + it("hub user can do anything we try", async () => { + for (const subject of ["*", "public", "global", ">", "hub", "test"]) { + for (const type of PUBSUB) { + expect( + await isAllowed({ user: { hub_id: "hub" }, type, subject }), + ).toBe(true); + } + } + }); +}); + +describe("test isAllowed for common subjects for projects and accounts", () => { + it("project user can't do random things", async () => { + for (const subject of ["*", "public", "global", ">", "hub", "test"]) { + for (const type of PUBSUB) { + expect(await isAllowed({ user: { project_id }, type, subject })).toBe( + false, + ); + } + } + }); + + it("project can publish to hub.project.project_id. but not subscribe", async () => { + // `hub.${userType}.${userId}.` + expect( + await isAllowed({ + user: { project_id }, + type: "pub", + subject: `hub.project.${project_id}.x`, + }), + ).toBe(true); + expect( + await isAllowed({ + user: { project_id }, + type: "sub", + subject: `hub.project.${project_id}.>`, + }), + ).toBe(false); + }); + + it("account can publish to hub.account.account_id. but not subscribe", async () => { + // `hub.${userType}.${userId}.` + expect( + await isAllowed({ + user: { account_id }, + type: "pub", + subject: `hub.account.${account_id}.x`, + }), + ).toBe(true); + expect( + await isAllowed({ + user: { account_id }, + type: "sub", + subject: `hub.account.${account_id}.>`, + }), + ).toBe(false); + }); + + it("account and project can publish to anything starting with _INBOX.", async () => { + expect( + await isAllowed({ + user: { account_id }, + type: "pub", + subject: `_INBOX.x`, + }), + ).toBe(true); + expect( + await isAllowed({ + user: { project_id }, + type: "pub", + subject: `_INBOX.x`, + }), + ).toBe(true); + }); + + it("account and project are allowed to subscribe to their custom inbox but not other inboxes", async () => { + expect( + await isAllowed({ + user: { account_id }, + type: "sub", + subject: inboxPrefix({ account_id }), + }), + ).toBe(true); + + expect( + await isAllowed({ + user: { project_id }, + type: "sub", + subject: inboxPrefix({ project_id }), + }), + ).toBe(true); + + expect( + await isAllowed({ + user: { project_id }, + type: "sub", + subject: inboxPrefix({ project_id: project_id2 }), + }), + ).toBe(false); + + expect( + await isAllowed({ + user: { account_id }, + type: "sub", + subject: inboxPrefix({ account_id: account_id2 }), + }), + ).toBe(false); + + // collab or not, account can't listen to inbox for a project: + (isCollaborator as jest.Mock).mockResolvedValue(false); + expect( + await isAllowed({ + user: { account_id }, + type: "sub", + subject: inboxPrefix({ project_id }), + }), + ).toBe(false); + + (isCollaborator as jest.Mock).mockResolvedValue(true); + expect( + await isAllowed({ + user: { account_id }, + type: "sub", + subject: inboxPrefix({ project_id }), + }), + ).toBe(false); + }); + + it("account and project can also subscribe to public.", async () => { + expect( + await isAllowed({ + user: { account_id }, + type: "sub", + subject: "public.>", + }), + ).toBe(true); + + expect( + await isAllowed({ + user: { project_id }, + type: "sub", + subject: "public.>", + }), + ).toBe(true); + }); + + it("account and project cannot publish to public.", async () => { + expect( + await isAllowed({ + user: { account_id }, + type: "pub", + subject: "public.version", + }), + ).toBe(false); + + expect( + await isAllowed({ + user: { project_id }, + type: "pub", + subject: "public.version", + }), + ).toBe(false); + }); +}); + +// `project.${project_id}.` and `*.project-${project_id}.>` +describe("test isAllowed for subjects special to projects", () => { + it("checks the special project subjects, which allow both pub and sub", async () => { + for (const type of PUBSUB) { + expect( + await isAllowed({ + user: { project_id }, + type, + subject: `project.${project_id}.`, + }), + ).toBe(true); + expect( + await isAllowed({ + user: { project_id }, + type, + subject: `foo.project-${project_id}.bar`, + }), + ).toBe(true); + } + }); +}); + +describe("test isAllowed for subjects special to accounts (similar to projects)", () => { + it("checks the special project subjects, which allow both pub and sub", async () => { + for (const type of PUBSUB) { + expect( + await isAllowed({ + user: { account_id }, + type, + subject: `account.${account_id}.`, + }), + ).toBe(true); + expect( + await isAllowed({ + user: { account_id }, + type, + subject: `foo.account-${account_id}.bar`, + }), + ).toBe(true); + } + }); +}); + +describe("test isAllowed for collaboration -- this is the most nontrivial one", () => { + it("verifies an account can access a project it collaborates on", async () => { + // Arrange: isCollaborator resolves to true + (isCollaborator as jest.Mock).mockResolvedValue(true); + + expect( + await isAllowed({ + user: { account_id }, + subject: `project.${project_id}.foo`, + type: "pub", + }), + ).toBe(true); + + expect(isCollaborator).toHaveBeenCalled(); + }); + + it("same account and project -- even if not collaborator still have permissions because of LRU cache!", async () => { + (isCollaborator as jest.Mock).mockResolvedValue(false); + + expect( + await isAllowed({ + user: { account_id }, + subject: `project.${project_id}.foo`, + type: "pub", + }), + ).toBe(true); + }); + + it("check on another project that not a collab on", async () => { + (isCollaborator as jest.Mock).mockResolvedValue(false); + + expect( + await isAllowed({ + user: { account_id }, + subject: `project.${project_id2}.foo`, + type: "pub", + }), + ).toBe(false); + }); +}); + +describe("tests system accounts", () => { + it("verifies a system user is not authenticated when there is no system account specified", async () => { + const socket = { handshake: { headers: { cookie: "Foo=bar;" } } }; + expect(async () => { + await getUser(socket); + }).rejects.toThrow("must set one of the following cookies"); + }); + + it("verifies system user works", async () => { + const socket = { handshake: { headers: { cookie: "System=pw;" } } }; + expect( + await getUser(socket, { + System: { + password: "pw", + user: { hub_id: "system" }, + }, + }), + ).toEqual({ hub_id: "system" }); + }); + + it("verifies system user works only with correct password", async () => { + const socket = { handshake: { headers: { cookie: "System=bad;" } } }; + expect(async () => { + await getUser(socket, { + System: { + password: "pw", + user: { hub_id: "system" }, + }, + }); + }).rejects.toThrow("invalid"); + }); +}); diff --git a/src/packages/server/conat/socketio/auth.ts b/src/packages/server/conat/socketio/auth.ts new file mode 100644 index 0000000000..0cb1c8bed4 --- /dev/null +++ b/src/packages/server/conat/socketio/auth.ts @@ -0,0 +1,332 @@ +import { inboxPrefix } from "@cocalc/conat/names"; +import { isValidUUID } from "@cocalc/util/misc"; +import isCollaborator from "@cocalc/server/projects/is-collaborator"; +import { getAccountIdFromRememberMe } from "@cocalc/server/auth/get-account"; +import { parse } from "cookie"; +import { getRememberMeHashFromCookieValue } from "@cocalc/server/auth/remember-me"; +import LRU from "lru-cache"; +import { conatPassword } from "@cocalc/backend/data"; +import { + API_COOKIE_NAME, + HUB_PASSWORD_COOKIE_NAME, + PROJECT_SECRET_COOKIE_NAME, + PROJECT_ID_COOKIE_NAME, + REMEMBER_ME_COOKIE_NAME, +} from "@cocalc/backend/auth/cookie-names"; +import { getAccountWithApiKey } from "@cocalc/server/api/manage"; +import { getProjectSecretToken } from "@cocalc/server/projects/control/secret-token"; +import { getAdmins } from "@cocalc/server/accounts/is-admin"; + +const COOKIES = `'${HUB_PASSWORD_COOKIE_NAME}', '${REMEMBER_ME_COOKIE_NAME}', ${API_COOKIE_NAME}, '${PROJECT_SECRET_COOKIE_NAME}' or '${PROJECT_ID_COOKIE_NAME}'`; + +export async function getUser( + socket, + systemAccounts?: { [cookieName: string]: { password: string; user: any } }, +): Promise { + if (!socket.handshake.headers.cookie) { + throw Error(`no auth cookie set; set one of ${COOKIES}`); + } + + const cookies = parse(socket.handshake.headers.cookie); + + if (systemAccounts != null) { + for (const cookieName in systemAccounts) { + if (cookies[cookieName] !== undefined) { + if (cookies[cookieName] == systemAccounts[cookieName].password) { + return systemAccounts[cookieName].user; + } else { + throw Error("invalid system account password"); + } + } + } + } + + if (cookies[HUB_PASSWORD_COOKIE_NAME]) { + if (cookies[HUB_PASSWORD_COOKIE_NAME] == conatPassword) { + return { hub_id: "hub" }; + } else { + throw Error(`invalid hub password`); + } + } + if (cookies[API_COOKIE_NAME]) { + // project or compute server or account + const user = await getAccountWithApiKey(cookies[API_COOKIE_NAME]!); + if (!user) { + throw Error("api key no longer valid"); + } + return user; + } + if (cookies[PROJECT_SECRET_COOKIE_NAME]) { + const project_id = cookies[PROJECT_ID_COOKIE_NAME]; + if (!project_id) { + throw Error( + `must specify project_id in the cookie ${PROJECT_ID_COOKIE_NAME}`, + ); + } + const secret = cookies[PROJECT_SECRET_COOKIE_NAME]; + if ((await getProjectSecretToken(project_id)) == secret) { + return { project_id: project_id! }; + } else { + throw Error("invalid secret token for project"); + // ONLY ENABLE THIS WHEN DOING DANGEROUS DEBUGGING + // TODO -- this is NOT secure! + // throw Error( + // `invalid secret token for project: ${JSON.stringify({ correct: await getProjectSecretToken(project_id), secret })}`, + // ); + } + } + + const value = cookies[REMEMBER_ME_COOKIE_NAME]; + if (!value) { + throw Error(`must set one of the following cookies: ${COOKIES}`); + } + const hash = getRememberMeHashFromCookieValue(value); + if (!hash) { + throw Error("invalid remember me cookie"); + } + const account_id = await getAccountIdFromRememberMe(hash); + if (!account_id) { + throw Error("remember me cookie expired"); + } + return { account_id }; +} + +const isAllowedCache = new LRU({ + max: 10000, + ttl: 1000 * 60, // 1 minute +}); + +export async function isAllowed({ + user, + subject, + type, +}: { + user?: CoCalcUser | null; + subject: string; + type: "sub" | "pub"; +}): Promise { + if (user == null || user?.error) { + // non-authenticated user -- allow NOTHING + return false; + } + const userType = getCoCalcUserType(user); + if (userType == "hub") { + // right now hubs have full permissions. + return true; + } + const userId = getCoCalcUserId(user); + const key = `${userType}-${userId}-${subject}-${type}`; + if (isAllowedCache.has(key)) { + return isAllowedCache.get(key)!; + } + + const common = checkCommonPermissions({ + userId, + userType, + user, + subject, + type, + }); + let allowed; + if (common != null) { + allowed = common; + } else if (userType == "project") { + allowed = isProjectAllowed({ project_id: userId, subject, type }); + } else if (userType == "account") { + allowed = await isAccountAllowed({ account_id: userId, subject, type }); + } else { + allowed = false; + } + isAllowedCache.set(key, allowed); + return allowed; +} + +export function checkCommonPermissions({ + user, + userType, + userId, + subject, + type, +}: { + user: CoCalcUser; + userType: "account" | "project"; + userId: string; + subject: string; + type: "sub" | "pub"; +}): null | boolean { + // can publish as *this user* to the hub's api's + if (subject.startsWith(`hub.${userType}.${userId}.`)) { + return type == "pub"; + } + + // everyone can publish to all inboxes. This seems like a major + // security risk, but with request/reply, the reply subject under + // _INBOX is a long random code that is only known for a moment + // by the sender and the service, so it is NOT a security risk. + if (type == "pub" && subject.startsWith("_INBOX.")) { + return true; + } + // custom inbox only for this user -- important for security, so we + // can only listen to messages for us, and not for anybody else. + if (type == "sub" && subject.startsWith(inboxPrefix(user))) { + return true; + } + + if (type == "sub" && subject.startsWith("public.")) { + return true; + } + + // no decision yet + return null; +} + +function isProjectAllowed({ + project_id, + subject, +}: { + project_id: string; + subject: string; + type: "sub" | "pub"; +}): boolean { + // pub and sub are the same + + if (subject.startsWith(`project.${project_id}.`)) { + return true; + } + // *.project-${project_id}.> + if (subject.split(".")[1] == `project-${project_id}`) { + return true; + } + + return false; +} + +async function isAccountAllowed({ + account_id, + subject, +}: { + account_id: string; + subject: string; + type: "sub" | "pub"; +}): Promise { + // pub and sub are the same + if (subject.startsWith(`account.${account_id}.`)) { + return true; + } + + const v = subject.split("."); + // *.account-${account_id}.> + if (v[1] == `account-${account_id}`) { + return true; + } + + if (v[0] == "sys") { + return (await getAdmins()).has(account_id); + } + + // account accessing a project + const project_id = extractProjectSubject(subject); + if (!project_id) { + return false; + } + return await isCollaborator({ account_id, project_id }); +} + +function extractProjectSubject(subject: string): string { + if (subject.startsWith("project.")) { + const project_id = subject.split(".")[1]; + if (isValidUUID(project_id)) { + return project_id; + } + return ""; + } + const v = subject.split("."); + if (v[1]?.startsWith("project-")) { + const project_id = v[1].slice("project-".length); + if (isValidUUID(project_id)) { + return project_id; + } + } + return ""; +} + +// A CoCalc User is (so far): a project or account or a hub +export type CoCalcUser = + | { + account_id: string; + project_id?: string; + hub_id?: string; + error?: string; + } + | { + account_id?: string; + project_id?: string; + hub_id: string; + error?: string; + } + | { + account_id?: string; + project_id: string; + hub_id?: string; + error?: string; + } + | { + account_id?: string; + project_id?: string; + hub_id?: string; + error: string; + }; + +export function getCoCalcUserType({ + account_id, + project_id, + hub_id, +}: CoCalcUser): "account" | "project" | "hub" { + if (account_id) { + if (project_id || hub_id) { + throw Error( + "exactly one of account_id or project_id or hub_id must be specified", + ); + } + return "account"; + } + if (project_id) { + if (hub_id) { + throw Error( + "exactly one of account_id or project_id or hub_id must be specified", + ); + } + return "project"; + } + if (hub_id) { + return "hub"; + } + throw Error("account_id or project_id or hub_id must be specified in User"); +} + +export function getCoCalcUserId({ + account_id, + project_id, + hub_id, +}: CoCalcUser): string { + if (account_id) { + if (project_id || hub_id) { + throw Error( + "exactly one of account_id or project_id or hub_id must be specified", + ); + } + return account_id; + } + if (project_id) { + if (hub_id) { + throw Error( + "exactly one of account_id or project_id or hub_id must be specified", + ); + } + return project_id; + } + if (hub_id) { + return hub_id; + } + throw Error("account_id or project_id or hub_id must be specified"); +} diff --git a/src/packages/server/conat/socketio/cluster.ts b/src/packages/server/conat/socketio/cluster.ts new file mode 100644 index 0000000000..7d26acfec1 --- /dev/null +++ b/src/packages/server/conat/socketio/cluster.ts @@ -0,0 +1,138 @@ +/* + +To start this: + + pnpm conat-server + +Run this to be able to use all the cores, since nodejs is (mostly) single threaded. +*/ + +import { init as createConatServer } from "@cocalc/conat/core/server"; +import cluster from "node:cluster"; +import * as http from "http"; +import { availableParallelism } from "os"; +import { + setupMaster as setupPrimarySticky, + setupWorker, +} from "@socket.io/sticky"; +import { createAdapter, setupPrimary } from "@socket.io/cluster-adapter"; +import { getUser, isAllowed } from "./auth"; +import { secureRandomString } from "@cocalc/backend/misc"; +import basePath from "@cocalc/backend/base-path"; +import port from "@cocalc/backend/port"; +import { + conatSocketioCount, + conatClusterPort, + conatClusterHealthPort, +} from "@cocalc/backend/data"; +import { loadConatConfiguration } from "../configuration"; +import { join } from "path"; + +// ensure conat logging, credentials, etc. is setup +import "@cocalc/backend/conat"; + +console.log(`* CONAT Core Pub/Sub Server on port ${port} *`); + +async function primary() { + console.log(`Socketio Server Primary pid=${process.pid} is running`); + + await loadConatConfiguration(); + + const httpServer = http.createServer(); + setupPrimarySticky(httpServer, { + loadBalancingMethod: "least-connection", + }); + + setupPrimary(); + cluster.setupPrimary({ serialization: "advanced" }); + httpServer.listen(getPort()); + + if (conatClusterHealthPort) { + console.log( + `starting /health socketio server on port ${conatClusterHealthPort}`, + ); + const healthServer = http.createServer(); + healthServer.listen(conatClusterHealthPort); + healthServer.on("request", (req, res) => { + // unhealthy if >3 deaths in 1 min + handleHealth(req, res, recentDeaths.length <= 3, "Too many worker exits"); + }); + } + + const numWorkers = conatSocketioCount + ? conatSocketioCount + : availableParallelism(); + const systemAccountPassword = await secureRandomString(32); + for (let i = 0; i < numWorkers; i++) { + cluster.fork({ SYSTEM_ACCOUNT_PASSWORD: systemAccountPassword }); + } + console.log({ numWorkers, port, basePath }); + + const recentDeaths: number[] = []; + cluster.on("exit", (worker) => { + if (conatClusterHealthPort) { + recentDeaths.push(Date.now()); + // Remove entries older than X seconds (e.g. 60s) + while (recentDeaths.length && recentDeaths[0] < Date.now() - 60_000) { + recentDeaths.shift(); + } + } + + console.log(`Worker ${worker.process.pid} died, so making a new one`); + cluster.fork(); + }); +} + +async function worker() { + console.log("BASE_PATH=", process.env.BASE_PATH); + await loadConatConfiguration(); + + const path = join(basePath, "conat"); + console.log(`Socketio Worker pid=${process.pid} started with path=${path}`); + + const httpServer = http.createServer(); + const id = `${cluster.worker?.id ?? ""}`; + const systemAccountPassword = process.env.SYSTEM_ACCOUNT_PASSWORD; + delete process.env.SYSTEM_ACCOUNT_PASSWORD; + + const conatServer = createConatServer({ + path, + httpServer, + id, + getUser, + isAllowed, + systemAccountPassword, + // port -- server needs to know implicitly to make a clients + port: getPort(), + cluster: true, + }); + conatServer.io.adapter(createAdapter()); + setupWorker(conatServer.io); +} + +function getPort() { + return conatClusterPort ? conatClusterPort : port; +} + +if (cluster.isPrimary) { + primary(); +} else { + worker(); +} + +function handleHealth( + req: http.IncomingMessage, + res: http.ServerResponse, + status: boolean, + msg?: string, +) { + if (req.method === "GET") { + if (status) { + res.statusCode = 200; + res.end("healthy"); + } else { + res.statusCode = 500; + res.end(msg || "Unhealthy"); + } + } +} diff --git a/src/packages/server/conat/socketio/index.ts b/src/packages/server/conat/socketio/index.ts new file mode 100644 index 0000000000..b920766d93 --- /dev/null +++ b/src/packages/server/conat/socketio/index.ts @@ -0,0 +1 @@ +export { init as initConatServer } from "./server"; diff --git a/src/packages/server/conat/socketio/server.ts b/src/packages/server/conat/socketio/server.ts new file mode 100644 index 0000000000..e75a1f6669 --- /dev/null +++ b/src/packages/server/conat/socketio/server.ts @@ -0,0 +1,130 @@ +/* +To start this standalone + + s = await require('@cocalc/server/conat/socketio').initConatServer() + +It will also get run integrated with the hub if the --conat-server option is passed in. + +Using valkey + + s1 = await require('@cocalc/server/conat/socketio').initConatServer({port:3000, valkey:'valkey://127.0.0.1:6379'}) + +or an example using an environment varaible and a password: + + CONAT_VALKEY=valkey://:test-password@127.0.0.1:6379 node + ... + > s1 = await require('@cocalc/server/conat/socketio').initConatServer({port:3000}) + + +and in another session: + + s2 = await require('@cocalc/server/conat/socketio').initConatServer({port:3001, valkey:'valkey://127.0.0.1:6379'}) + +Then make a client connected to each: + + c1 = require('@cocalc/conat/core/client').connect('http://localhost:3000'); + c2 = require('@cocalc/conat/core/client').connect('http://localhost:3001'); + +*/ + +import { + init as createConatServer, + type Options, +} from "@cocalc/conat/core/server"; +import { getUser, isAllowed } from "./auth"; +import { secureRandomString } from "@cocalc/backend/misc"; +import { + conatValkey, + conatSocketioCount, + valkeyPassword, + conatClusterPort, +} from "@cocalc/backend/data"; +import basePath from "@cocalc/backend/base-path"; +import port from "@cocalc/backend/port"; +import { join } from "path"; +import startCluster from "./start-cluster"; +import { getLogger } from "@cocalc/backend/logger"; +import "@cocalc/backend/conat"; + +const logger = getLogger("conat-server"); + +export async function init(options: Partial = {}) { + logger.debug("init"); + console.log({ conatClusterPort, conatSocketioCount }); + + if (conatClusterPort) { + const mesg = `Conat cluster port is set so we spawn a cluster listening on port ${conatClusterPort}, instead of an in-process conat server`; + console.log(mesg); + logger.debug(mesg); + startCluster(); + return; + } + + let valkey: undefined | string | any = undefined; + if (valkeyPassword) { + // only hope is making valkey an object + if (conatValkey) { + if (conatValkey.startsWith("sentinel://")) { + valkey = parseSentinelConfig(conatValkey, valkeyPassword); + } else { + valkey = parseValkeyConfigString(conatValkey, valkeyPassword); + } + } + } else if (conatValkey) { + valkey = conatValkey; + } + + const opts = { + getUser, + isAllowed, + systemAccountPassword: await secureRandomString(64), + valkey, + path: join(basePath, "conat"), + port, + ...options, + }; + + if (!conatSocketioCount || conatSocketioCount <= 1) { + return createConatServer(opts); + } else { + // spawn conatSocketioCount subprocesses listening on random available ports + // and all connected to valkey. Proxy traffic to them. + throw Error(`not implemented -- socket count = ${conatSocketioCount}`); + } +} + +export function parseValkeyConfigString(conatValkey: string, password: string) { + const i = conatValkey.lastIndexOf("/"); + const x = conatValkey.slice(i + 1); + const v = x.split(":"); + return { + host: v[0] ?? "localhost", + port: v[1] ? parseInt(v[1]) : 6379, + password, + }; +} + +// E.g., input: sentinel://valkey-sentinel-0,valkey-sentinel-1,valkey-sentinel-2 + +export function parseSentinelConfig(conatValkey: string, password: string) { + /* + name: "cocalc", + sentinelPassword: password, + password, + sentinels: [0, 1, 2].map((i) => ({ + host: `valkey-sentinel-${i}`, + port: 26379, + })), + */ + const config = { + name: "cocalc", + sentinelPassword: password, + password, + sentinels: [] as { host: string; port: number }[], + }; + for (const sentinel of conatValkey.split("sentinel://")[1].split(",")) { + const [host, port = "26379"] = sentinel.split(":"); + config.sentinels.push({ host, port: parseInt(port) }); + } + return config; +} diff --git a/src/packages/server/conat/socketio/start-cluster.ts b/src/packages/server/conat/socketio/start-cluster.ts new file mode 100644 index 0000000000..3d58bb67c9 --- /dev/null +++ b/src/packages/server/conat/socketio/start-cluster.ts @@ -0,0 +1,61 @@ +import { spawn, ChildProcess } from "node:child_process"; +import { join } from "path"; +import { conatSocketioCount, conatClusterPort } from "@cocalc/backend/data"; +import basePath from "@cocalc/backend/base-path"; + +const servers: { close: Function }[] = []; + +export default function startCluster({ + port = conatClusterPort, + numWorkers = conatSocketioCount, +}: { port?: number; numWorkers?: number } = {}) { + // spawn valkey-server listening on port running in a mode where + // data is never saved to disk using the nodejs spawn command: + // // Start valkey-server with in-memory only, no persistence + const child: ChildProcess = spawn( + process.argv[0], + [join(__dirname, "cluster.js")], + { + stdio: "inherit", + detached: false, + cwd: __dirname, + env: { + ...process.env, + PORT: `${port}`, + CONAT_SOCKETIO_COUNT: `${numWorkers}`, + BASE_PATH: basePath, + }, + }, + ); + + let closed = false; + const close = () => { + if (closed) return; + closed = true; + if (!child?.pid) return; + try { + process.kill(child.pid, "SIGKILL"); + } catch { + // already dead or not found + } + }; + + const server = { + close, + }; + servers.push(server); + return server; +} + +process.once("exit", () => { + for (const { close } of servers) { + try { + close(); + } catch {} + } +}); +["SIGINT", "SIGTERM", "SIGQUIT"].forEach((sig) => { + process.once(sig, () => { + process.exit(); + }); +}); diff --git a/src/packages/server/jest.config.js b/src/packages/server/jest.config.js index 3ea89e18a9..763f4887ba 100644 --- a/src/packages/server/jest.config.js +++ b/src/packages/server/jest.config.js @@ -1,7 +1,7 @@ /** @type {import('ts-jest').JestConfigWithTsJest} */ module.exports = { - preset: 'ts-jest', - testEnvironment: 'node', - setupFiles: ['./test/setup.js'], // Path to your setup file - testMatch: ['**/?(*.)+(spec|test).ts?(x)'], + preset: "ts-jest", + testEnvironment: "node", + setupFiles: ["./test/setup.js"], // Path to your setup file + testMatch: ["**/?(*.)+(spec|test).ts?(x)"], }; diff --git a/src/packages/server/nats/api/index.ts b/src/packages/server/nats/api/index.ts deleted file mode 100644 index 6ae0921e9f..0000000000 --- a/src/packages/server/nats/api/index.ts +++ /dev/null @@ -1,230 +0,0 @@ -/* -This is meant to be similar to the nexts pages http api/v2, but using NATS instead of HTTPS. - -To do development: - -1. Turn off nats-server handling for the hub by sending this message from a browser as an admin: - - await cc.client.nats_client.hub.system.terminate({service:'api'}) - -NOTE: there's no way to turn the auth back on in the hub, so you'll have to restart -your dev hub after doing the above. - -2. Run this script at the terminal: - - echo "require('@cocalc/server/nats').default()" | COCALC_MODE='single-user' DEBUG_CONSOLE=yes DEBUG=cocalc:* node - - -3. Optional: start more servers -- requests get randomly routed to exactly one of them: - - echo "require('@cocalc/server/nats').default()" | COCALC_MODE='single-user' DEBUG_CONSOLE=yes DEBUG=cocalc:* node - echo "require('@cocalc/server/nats').default()" | COCALC_MODE='single-user' DEBUG_CONSOLE=yes DEBUG=cocalc:* node - - -To make use of this from a browser: - - await cc.client.nats_client.hub.system.getCustomize(['siteName']) - -or - - await cc.client.nats_client.callHub({name:"system.getCustomize", args:[['siteName']]}) - -When you make changes, just restart the above. All clients will instantly -use the new version after you restart, and there is no need to restart the hub -itself or any clients. - -To view all requests (and replies) in realtime: - - nats sub 'hub.*.*.api' --match-replies - -And remember to use the nats command, do "pnpm nats-cli" from cocalc/src. -*/ - -import { JSONCodec } from "nats"; -import getLogger from "@cocalc/backend/logger"; -import { type HubApi, getUserId, transformArgs } from "@cocalc/nats/hub-api"; -import { getConnection } from "@cocalc/backend/nats"; -import userIsInGroup from "@cocalc/server/accounts/is-in-group"; -import { terminate as terminateDatabase } from "@cocalc/database/nats/changefeeds"; -import { terminate as terminateChangefeedServer } from "@cocalc/nats/changefeed/server"; -import { Svcm } from "@nats-io/services"; -import { terminate as terminateAuth } from "@cocalc/server/nats/auth"; -import { terminate as terminateTieredStorage } from "@cocalc/server/nats/tiered-storage/api"; -import { respondMany } from "@cocalc/nats/service/many"; -import { delay } from "awaiting"; -import { waitUntilConnected } from "@cocalc/nats/util"; - -const MONITOR_INTERVAL = 30000; - -const logger = getLogger("server:nats:api"); - -const jc = JSONCodec(); - -export function initAPI() { - mainLoop(); -} - -let terminate = false; -async function mainLoop() { - let d = 3000; - let lastStart = 0; - while (!terminate) { - try { - lastStart = Date.now(); - await serve(); - } catch (err) { - logger.debug(`hub nats api service error -- ${err}`); - if (Date.now() - lastStart >= 30000) { - // it ran for a while, so no delay - logger.debug(`will restart immediately`); - d = 3000; - } else { - // it crashed quickly, so delay! - d = Math.min(20000, d * 1.25 + Math.random()); - logger.debug(`will restart in ${d}ms`); - await delay(d); - } - } - } -} - -async function serviceMonitor({ nc, api, subject }) { - while (!terminate) { - logger.debug(`serviceMonitor: waiting ${MONITOR_INTERVAL}ms`); - await delay(MONITOR_INTERVAL); - try { - await waitUntilConnected(); - await nc.request(subject, jc.encode({ name: "ping" }), { - timeout: 7500, - }); - logger.debug("serviceMonitor: ping succeeded"); - } catch (err) { - logger.debug( - `serviceMonitor: ping failed, so restarting service -- ${err}`, - ); - api.stop(); - return; - } - } -} - -async function serve() { - const subject = "hub.*.*.api"; - logger.debug(`initAPI -- subject='${subject}', options=`, { - queue: "0", - }); - const nc = await getConnection(); - // @ts-ignore - const svcm = new Svcm(nc); - - await waitUntilConnected(); - const service = await svcm.add({ - name: "hub-server", - version: "0.1.0", - description: "CoCalc Hub Server", - }); - - const api = service.addEndpoint("api", { subject }); - serviceMonitor({ api, subject, nc }); - await listen({ api, subject }); -} - -async function listen({ api, subject }) { - for await (const mesg of api) { - const request = jc.decode(mesg.data) ?? ({} as any); - if (request.name == "system.terminate") { - // special hook so admin can terminate handling. This is useful for development. - const { account_id } = getUserId(mesg.subject); - if (!(!!account_id && (await userIsInGroup(account_id, "admin")))) { - mesg.respond(jc.encode({ error: "only admin can terminate" })); - continue; - } - // TODO: could be part of handleApiRequest below, but done differently because - // one case halts this loop - const { service } = request.args[0] ?? {}; - logger.debug(`Terminate service '${service}'`); - if (service == "db") { - terminateDatabase(); - mesg.respond(jc.encode({ status: "terminated", service })); - continue; - } else if (service == "auth") { - terminateAuth(); - mesg.respond(jc.encode({ status: "terminated", service })); - continue; - } else if (service == "tiered-storage") { - terminateTieredStorage(); - mesg.respond(jc.encode({ status: "terminated", service })); - continue; - } else if (service == "changefeeds") { - terminateChangefeedServer(); - mesg.respond(jc.encode({ status: "terminated", service })); - continue; - } else if (service == "api") { - // special hook so admin can terminate handling. This is useful for development. - console.warn("TERMINATING listening on ", subject); - logger.debug("TERMINATING listening on ", subject); - terminate = true; - mesg.respond(jc.encode({ status: "terminated", service })); - api.stop(); - return; - } else { - mesg.respond(jc.encode({ error: `Unknown service ${service}` })); - } - } else { - // we explicitly do NOT await this, since we want this hub server to handle - // potentially many messages at once, not one at a time! - handleApiRequest({ request, mesg }); - } - } -} - -async function handleApiRequest({ request, mesg }) { - let resp; - try { - const { account_id, project_id } = getUserId(mesg.subject); - const { name, args } = request as any; - logger.debug("handling hub.api request:", { - account_id, - project_id, - name, - }); - resp = (await getResponse({ name, args, account_id, project_id })) ?? null; - } catch (err) { - resp = { error: `${err}` }; - } - try { - await respondMany({ mesg, data: jc.encode(resp) }); - } catch (err) { - // there's nothing we can do here, e.g., maybe NATS just died. - logger.debug( - `WARNING: error responding to hub.api request (client will receive no response) -- ${err}`, - ); - } -} - -import * as purchases from "./purchases"; -import * as db from "./db"; -import * as system from "./system"; -import * as projects from "./projects"; - -export const hubApi: HubApi = { - system, - projects, - db, - purchases, -}; - -async function getResponse({ name, args, account_id, project_id }) { - const [group, functionName] = name.split("."); - const f = hubApi[group]?.[functionName]; - if (f == null) { - throw Error(`unknown function '${name}'`); - } - const args2 = await transformArgs({ - name, - args, - account_id, - project_id, - }); - return await f(...args2); -} diff --git a/src/packages/server/nats/api/system.ts b/src/packages/server/nats/api/system.ts deleted file mode 100644 index 7ee5d275fa..0000000000 --- a/src/packages/server/nats/api/system.ts +++ /dev/null @@ -1,62 +0,0 @@ -import getCustomize from "@cocalc/database/settings/customize"; -export { getCustomize }; -import { record_user_tracking } from "@cocalc/database/postgres/user-tracking"; -import { db } from "@cocalc/database"; -import manageApiKeys from "@cocalc/server/api/manage"; -export { manageApiKeys }; -import { type UserSearchResult } from "@cocalc/util/db-schema/accounts"; -import isAdmin from "@cocalc/server/accounts/is-admin"; -import search from "@cocalc/server/accounts/search"; -export { getNames } from "@cocalc/server/accounts/get-name"; - -export function ping() { - return { now: Date.now() }; -} - -export async function terminate() {} - -export async function userTracking({ - event, - value, - account_id, -}: { - event: string; - value: object; - account_id?: string; -}): Promise { - await record_user_tracking(db(), account_id!, event, value); -} - -export { - generateUserAuthToken, - revokeUserAuthToken, -} from "@cocalc/server/auth/auth-token"; - -export async function userSearch({ - account_id, - query, - limit, - admin, - only_email, -}: { - account_id?: string; - query: string; - limit?: number; - admin?: boolean; - only_email?: boolean; -}): Promise { - if (!account_id) { - throw Error("You must be signed in to search for users."); - } - if (admin) { - if (!(await isAdmin(account_id))) { - throw Error("Must be an admin to do admin search."); - } - } else { - if (limit != null && limit > 50) { - // hard cap at 50... (for non-admin) - limit = 50; - } - } - return await search({ query, limit, admin, only_email }); -} diff --git a/src/packages/server/nats/auth/index.ts b/src/packages/server/nats/auth/index.ts deleted file mode 100644 index 4c4887a5e9..0000000000 --- a/src/packages/server/nats/auth/index.ts +++ /dev/null @@ -1,274 +0,0 @@ -/* -Implementation of Auth Callout for NATS - -DEPLOYMENT: - -Run as many of these as you want -- the load gets randomly spread across all of them. -They just need access to the database. - -There is some nontrivial compute associated with handling each auth, due to: - - - 1000 rounds of sha512 for the remember_me cookie takes time - - encoding/encrypting/decoding/decrypting JWT stuff with NATS takes maybe 50ms of CPU. - -The main "weird" thing about this is that when a connection is being authenticated, -we have to decide on its *exact* permissions once-and-for all at that point in time. -This means browser clients have to reconnect if they want to communicate with a project -they didn't explicit authenticate to. - -AUTH CALLOUT - -At least one of these cocalc servers (which relies on the database) *must* -be available to handle every user connection, unlike with decentralized PKI auth. -It also makes banning users a bit more complicated. - -DOCS: - -- https://docs.nats.io/running-a-nats-service/configuration/securing_nats/auth_callout - -- https://github.com/nats-io/nats-architecture-and-design/blob/main/adr/ADR-26.md - -- https://natsbyexample.com/examples/auth/callout/cli - -- https://www.youtube.com/watch?v=VvGxrT-jv64 - - -DEVELOPMENT - -1. From the browser, turn off the nats auth that is being served by your development hub -by sending this message from a browser as an admin: - - await cc.client.nats_client.hub.system.terminate({service:'auth'}) - -2. Run this code in nodejs: - - x = await require('@cocalc/server/nats/auth').init() - - -NOTE: there's no way to turn the auth back on in the hub, so you'll have to restart -your dev hub after doing the above. - - -WHY NOT DECENTRALIZED AUTH? - -I wish I knew the following earlier, as it would have saved me at least a -week of work... - -We *fully* implemented decentralized auth first using JWT's, but it DOES NOT -SCALE! The problem is that we need potentially dozens of pub/sub rules for each -user, so that's too much information to put in a client JWT cookie, so we -*must* use signing keys. Thus the permissions information for every user goes -into one massive account key blob, and a tiny signed JWT goes to each browser. -This is very nice because permissions can be dynamically updated at any time, -and everybody's permissions are known to NATS without cocalc's database having -to be consulted at all (that said, with multiple nats servers, I am worries the -permissions update would take too long). SADLY, this doesn't scale! -Every time we make a change, the account key has to be updated, and only -a few hundred (or thousand) -users are enough to make the key too big to fit in a message. -Also, each update would take at least a second. Now imagine 150 students in -a class all signing in at once, and it taking over 150 seconds just to -process auth, and you can see this is a nonstarter. -Decentralized auth could work if each cocalc user had a different -account, but... that doesn't work either, since import/export doesn't -really work for jetstream... and setting up all the -import/export would be a nightmare, and probaby much more complicated. - -NOTE: There is one approach to decentralized auth that doesn't obviously fail, -but it would require a separate websocket connection for each project and maybe -some mangling of auth cookies in the proxy server. That said, it still relies -on using command line nsc with pull/push, which feels very slow and brittle. -Using a separate connection for each project is also just really bad practice. -*/ - -import { Svcm } from "@nats-io/services"; -import { getConnection } from "@cocalc/backend/nats"; -import type { NatsConnection } from "@nats-io/nats-core"; -import { - natsAuthCalloutNSeed, - natsAuthCalloutXSeed, -} from "@cocalc/backend/data"; -import { fromPublic, fromSeed } from "@nats-io/nkeys"; -import { - decode as decodeJwt, - encodeAuthorizationResponse, - encodeUser, -} from "@nats-io/jwt"; -import getLogger from "@cocalc/backend/logger"; -import { getUserPermissions } from "./permissions"; -import { validate } from "./validate"; -import adminAlert from "@cocalc/server/messages/admin-alert"; - - -// we put a per-connection limit on subscription to hopefully avoid -// some potential DOS situations. For reference each open file -// takes up to 15 subs (3 for a txt file, ~15 for a jupyter notebook). -// WARNING: I do not think this does anything at all: -const MAX_SUBSCRIPTIONS = 1500; -//const MAX_SUBSCRIPTIONS = 50; - -// some high but nontrivial limit on MB per second for each client -// WARNING: I do not think this does anything at all: -const MAX_BYTES_SECOND = 100 * 1000000; -//const MAX_BYTES_SECOND = 1000000; - -// ADMIN -- use `pnpm nats-cli-sys` then `nats server report connections` -// to see the number of connections by each user. - -const logger = getLogger("server:nats:auth-callout"); - -let api: any | null = null; -export async function init() { - logger.debug("init"); - // coerce to NatsConnection is to workaround a bug in the - // typescript libraries for nats, which might disappear at some point. - const nc = (await getConnection()) as NatsConnection; - const svcm = new Svcm(nc); - - const service = await svcm.add({ - name: "auth", - version: "0.0.1", - description: "CoCalc auth callout service", - // all auth callout handlers randomly take turns authenticating users - queue: "q", - }); - const g = service.addGroup("$SYS").addGroup("REQ").addGroup("USER"); - api = g.addEndpoint("AUTH"); - const encoder = new TextEncoder(); - - const xkp = fromSeed(encoder.encode(natsAuthCalloutXSeed)); - listen(api, xkp); - - return { - service, - nc, - close: () => { - api.stop(); - }, - }; -} - -export function terminate() { - api?.stop(); -} - -//const SESSION_EXPIRE_MS = 1000 * 60 * 60 * 12; - -async function listen(api, xkp) { - logger.debug("listening..."); - try { - for await (const mesg of api) { - // do NOT await this - handleRequest(mesg, xkp); - } - } catch (err) { - logger.debug("WARNING: Problem with auth callout", err); - // restart? I don't know why this would ever fail assuming - // our code isn't buggy, hence alert if this ever happens: - adminAlert({ - subject: "NATS auth-callout service crashed", - body: `A nats auth callout service crashed with the following error:\n\n${err}\n\nWilliam thinks this is impossible and will never happen, so investigate. This problem could cause all connections to cocalc to fail, and would be fixable by restarting certain hubs.`, - }); - } -} - -async function handleRequest(mesg, xkp) { - const t0 = Date.now(); - try { - const requestJwt = getRequestJwt(mesg, xkp); - const requestClaim = decodeJwt(requestJwt) as any; - logger.debug("handleRequest", requestClaim.nats.connect_opts.name); - const userNkey = requestClaim.nats.user_nkey; - const serverId = requestClaim.nats.server_id; - const { pub, sub } = await getPermissions(requestClaim.nats.connect_opts); - const user = fromPublic(userNkey); - const server = fromPublic(serverId.name); - const encoder = new TextEncoder(); - const issuer = fromSeed(encoder.encode(natsAuthCalloutNSeed)); - const userName = requestClaim.nats.connect_opts.user; - const opts = { aud: "COCALC" }; - const jwt = await encodeUser( - userName, - user, - issuer, - { - pub, - sub, - locale: Intl.DateTimeFormat().resolvedOptions().timeZone, - // I don't think the data and subs limits actually do anything at all. - // bytes per second - data: MAX_BYTES_SECOND, - // total number of simultaneous subscriptions - subs: MAX_SUBSCRIPTIONS, - }, - opts, - ); - const data = { jwt }; - const authResponse = await encodeAuthorizationResponse( - user, - server, - issuer, - data, - opts, - ); - const xkey = mesg.headers.get("Nats-Server-Xkey"); - let signedResponse; - if (xkey) { - signedResponse = xkp.seal(encoder.encode(authResponse), xkey); - } else { - signedResponse = encoder.encode(authResponse); - } - - mesg.respond(signedResponse); - } catch (err) { - // TODO: send fail response (?) - logger.debug(`Warning - ${err}`); - } finally { - logger.debug( - `time to handle one auth request completely: ${Date.now() - t0}ms`, - ); - } -} - -function getRequestJwt(mesg, xkp): string { - const xkey = mesg.headers.get("Nats-Server-Xkey"); - let data; - if (xkey) { - // encrypted - // we have natsAuthCalloutXSeedPath above. So have enough info to decrypt. - data = xkp.open(mesg.data, xkey); - } else { - // not encrypted - data = mesg.data; - } - const decoder = new TextDecoder("utf-8"); - return decoder.decode(data); -} - -async function getPermissions({ - auth_token, - name, -}: { - // auth token: - // - remember me - // - api key - // - project secret - auth_token?: string; - name?: string; -}) { - if (!name) { - throw Error("name must be specified"); - } - const { - account_id, - project_id, - project_ids: requested_project_ids, - } = JSON.parse(name) ?? {}; - const { project_ids } = await validate({ - account_id, - project_id, - auth_token, - requested_project_ids, - }); - return getUserPermissions({ account_id, project_id, project_ids }); -} diff --git a/src/packages/server/nats/auth/permissions.ts b/src/packages/server/nats/auth/permissions.ts deleted file mode 100644 index c37669611a..0000000000 --- a/src/packages/server/nats/auth/permissions.ts +++ /dev/null @@ -1,184 +0,0 @@ -import { inboxPrefix } from "@cocalc/nats/names"; -import getLogger from "@cocalc/backend/logger"; -import { isValidUUID } from "@cocalc/util/misc"; - -const logger = getLogger("server:nats:auth:permissions"); - -export function getUserPermissions({ - project_id, - account_id, - project_ids, -}: CoCalcUser & { project_ids?: string[] }) { - logger.debug("getUserPermissions", { - account_id, - project_id, - project_ids, - }); - if (project_id) { - if (!isValidUUID(project_id)) { - throw Error(`invalid project_id ${project_id}`); - } - // project_ids are ignored in this case - return projectPermissions(project_id); - } else if (account_id) { - if (!isValidUUID(account_id)) { - throw Error(`invalid account_id ${account_id}`); - } - const { pub, sub } = accountPermissions(account_id); - if (project_ids) { - for (const project_id of project_ids) { - if (!isValidUUID(project_id)) { - throw Error(`invalid project_id ${project_id}`); - } - const x = projectPermissions(project_id); - pub.allow.push(...x.pub.allow); - sub.allow.push(...x.sub.allow); - pub.deny.push(...x.pub.deny); - sub.deny.push(...x.sub.deny); - } - } - // uniq because there is a little overlap - return { - pub: { allow: uniq(pub.allow), deny: uniq(pub.deny) }, - sub: { allow: uniq(sub.allow), deny: uniq(sub.deny) }, - }; - } else { - throw Error("account_id or project_id must be specified"); - } -} - -function uniq(v: string[]): string[] { - return Array.from(new Set(v)); -} - -function commonPermissions(cocalcUser) { - const pub = { allow: [] as string[], deny: [] as string[] }; - const sub = { allow: [] as string[], deny: [] as string[] }; - const userId = getCoCalcUserId(cocalcUser); - if (!isValidUUID(userId)) { - throw Error("must be a valid uuid"); - } - const userType = getCoCalcUserType(cocalcUser); - - // can talk as *only this user* to the hub's api's - pub.allow.push(`hub.${userType}.${userId}.>`); - // everyone can publish to all inboxes. This seems like a major - // security risk, but with request/reply, the reply subject under - // _INBOX is a long random code that is only known for a moment - // by the sender and the service, so I think it is NOT a security risk. - pub.allow.push("_INBOX.>"); - - // custom inbox only for this user -- critical for security, so we - // can only listen to messages for us, and not for anybody else. - sub.allow.push(inboxPrefix(cocalcUser) + ".>"); - // access to READ the public system info kv store. - sub.allow.push("public.>"); - - // get info about jetstreams - pub.allow.push("$JS.API.INFO"); - // the public jetstream: this makes it available *read only* to all accounts and projects. - pub.allow.push("$JS.API.*.*.public"); - pub.allow.push("$JS.API.*.*.public.>"); - pub.allow.push("$JS.API.CONSUMER.MSG.NEXT.public.>"); - - // everyone can ack messages -- this publish to something like this - // $JS.ACK.account-6aae57c6-08f1-4bb5-848b-3ceb53e61ede.lZiQnTzW.11.1.98.1743611921171669063.0 - // which contains a random string, so there is no danger letting anyone publish to this. - pub.allow.push("$JS.ACK.>"); - - // microservices info api -- **TODO: security concerns!?** - // Please don't tell me I have to name all microservice identically :-( - sub.allow.push("$SRV.>"); - pub.allow.push("$SRV.>"); - - // so client can find out what they can pub/sub to... - pub.allow.push("$SYS.REQ.USER.INFO"); - return { pub, sub }; -} - -function projectPermissions(project_id: string) { - const { pub, sub } = commonPermissions({ project_id }); - - pub.allow.push(`project.${project_id}.>`); - sub.allow.push(`project.${project_id}.>`); - - pub.allow.push(`*.project-${project_id}.>`); - sub.allow.push(`*.project-${project_id}.>`); - - // The unique project-wide kv store: - pub.allow.push(`$JS.*.*.*.KV_project-${project_id}`); - pub.allow.push(`$JS.*.*.*.KV_project-${project_id}.>`); - - // this FC is needed for "flow control" - without this, you get random hangs forever at scale! - pub.allow.push(`$JS.FC.KV_project-${project_id}.>`); - - // The unique project-wide stream: - pub.allow.push(`$JS.*.*.*.project-${project_id}`); - pub.allow.push(`$JS.*.*.*.project-${project_id}.>`); - pub.allow.push(`$JS.*.*.*.*.project-${project_id}.>`); - return { pub, sub }; -} - -function accountPermissions(account_id: string) { - const { pub, sub } = commonPermissions({ account_id }); - sub.allow.push(`*.account-${account_id}.>`); - pub.allow.push(`*.account-${account_id}.>`); - - // the account-specific kv store: - pub.allow.push(`$JS.*.*.*.KV_account-${account_id}`); - pub.allow.push(`$JS.*.*.*.KV_account-${account_id}.>`); - - // the account-specific stream: - // (not used yet at all!) - pub.allow.push(`$JS.*.*.*.account-${account_id}`); - pub.allow.push(`$JS.*.*.*.account-${account_id}.>`); - pub.allow.push(`$JS.*.*.*.*.account-${account_id}`); - pub.allow.push(`$JS.*.*.*.*.account-${account_id}.>`); - - sub.allow.push(`account.${account_id}.>`); - pub.allow.push(`account.${account_id}.>`); - - // this FC is needed for "flow control" - without this, you get random hangs forever at scale! - pub.allow.push(`$JS.FC.KV_account-${account_id}.>`); - return { pub, sub }; -} - -// A CoCalc User is (so far): a project or account or a hub (not covered here). -type CoCalcUser = - | { - account_id: string; - project_id?: string; - } - | { - account_id?: string; - project_id: string; - }; - -function getCoCalcUserType({ - account_id, - project_id, -}: CoCalcUser): "account" | "project" { - if (account_id) { - if (project_id) { - throw Error("exactly one of account_id or project_id must be specified"); - } - return "account"; - } - if (project_id) { - return "project"; - } - throw Error("account_id or project_id must be specified"); -} - -function getCoCalcUserId({ account_id, project_id }: CoCalcUser): string { - if (account_id) { - if (project_id) { - throw Error("exactly one of account_id or project_id must be specified"); - } - return account_id; - } - if (project_id) { - return project_id; - } - throw Error("account_id or project_id must be specified"); -} diff --git a/src/packages/server/nats/auth/stress.ts b/src/packages/server/nats/auth/stress.ts deleted file mode 100644 index bf1222db2e..0000000000 --- a/src/packages/server/nats/auth/stress.ts +++ /dev/null @@ -1,15 +0,0 @@ -/* -Tools for stress testing nats so we understand it better. - -NOTHING USEFUL HERE NOW -*/ - -export function intToUuid(n) { - const base8 = n.toString(8); - const padded = base8.padStart(32, "0"); - return `${padded.slice(0, 8)}-${padded.slice(8, 12)}-${padded.slice(12, 16)}-${padded.slice(16, 20)}-${padded.slice(20, 32)}`; -} - -export function progress({ n, stop }) { - console.log(`${n}/${stop}`); -} diff --git a/src/packages/server/nats/auth/validate.ts b/src/packages/server/nats/auth/validate.ts deleted file mode 100644 index c1eeeaae62..0000000000 --- a/src/packages/server/nats/auth/validate.ts +++ /dev/null @@ -1,88 +0,0 @@ -import getPool from "@cocalc/database/pool"; -import { getAccountWithApiKey } from "@cocalc/server/api/manage"; -import { subsetCollaboratorMulti } from "@cocalc/server/projects/is-collaborator"; -import { getAccountIdFromRememberMe } from "@cocalc/server/auth/get-account"; -import { getRememberMeHashFromCookieValue } from "@cocalc/server/auth/remember-me"; - -// if throw error or not return true, then validation fails. -// success = NOT throwing error and returning true. - -export async function validate({ - account_id, - project_id, - requested_project_ids, - auth_token, -}: { - account_id?: string; - project_id?: string; - requested_project_ids?: string[]; - auth_token?: string; -}): Promise<{ project_ids?: string[] }> { - if (account_id && project_id) { - throw Error("exactly one of account_id and project_id must be specified"); - } - if (!auth_token) { - throw Error("auth_token must be specified"); - } - - // are they who they say they are? - await assertValidUser({ account_id, project_id, auth_token }); - - // we now know that auth_token provides they are either project_id or account_id. - // what about requested_project_ids? - if ( - !requested_project_ids || - requested_project_ids.length == 0 || - project_id - ) { - // none requested or is a project - return {}; - } - - if (!account_id) { - throw Error("bug"); - } - const project_ids = await subsetCollaboratorMulti({ - account_id, - project_ids: requested_project_ids, - }); - return { project_ids }; -} - -async function assertValidUser({ auth_token, project_id, account_id }) { - if (auth_token?.startsWith("sk-") || auth_token?.startsWith("sk_")) { - // auth_token is presumably an api key - const a = await getAccountWithApiKey(auth_token); - if (project_id && a?.project_id == project_id) { - return; - } else if (account_id && a?.account_id == account_id) { - return; - } - throw Error( - `auth_token valid for ${JSON.stringify(a)} by does not match ${project_id} or ${account_id}`, - ); - } - if (project_id) { - if ((await getProjectSecretToken(project_id)) == auth_token) { - return; - } - } - if (account_id) { - // maybe auth_token is a valid remember me browser cookie? - const hash = getRememberMeHashFromCookieValue(auth_token); - if (hash && account_id == (await getAccountIdFromRememberMe(hash))) { - return; - } - } - // nothing above matches, so FAIL! - throw Error("invalid auth_token"); -} - -async function getProjectSecretToken(project_id): Promise { - const pool = getPool(); - const { rows } = await pool.query( - "select status#>'{secret_token}' as secret_token from projects where project_id=$1", - [project_id], - ); - return rows[0]?.secret_token; -} diff --git a/src/packages/server/nats/configuration.ts b/src/packages/server/nats/configuration.ts deleted file mode 100644 index da067389ac..0000000000 --- a/src/packages/server/nats/configuration.ts +++ /dev/null @@ -1,50 +0,0 @@ -/* -Load NATS configuration from the database, in case anything is set there. -*/ - -import getPool from "@cocalc/database/pool"; -import { - setNatsPassword, - setNatsServer, - setNatsPort, - setNatsWebsocketPort, - setNatsAuthCalloutNSeed, - setNatsAuthCalloutXSeed, -} from "@cocalc/backend/data"; - -export async function loadNatsConfiguration() { - const pool = getPool(); - const { rows } = await pool.query( - "SELECT name, value FROM server_settings WHERE name=ANY($1)", - [ - [ - "nats_password", - "nats_auth_nseed", - "nats_auth_xseed", - "nats_port", - "nats_ws_port", - "nats_server", - ], - ], - ); - for (const { name, value } of rows) { - if (!value) { - continue; - } - if (name == "nats_password") { - setNatsPassword(value.trim()); - } else if (name == "nats_auth_nseed") { - setNatsAuthCalloutNSeed(value.trim()); - } else if (name == "nats_auth_xseed") { - setNatsAuthCalloutXSeed(value.trim()); - } else if (name == "nats_server") { - setNatsServer(value.trim()); - } else if (name == "nats_port") { - setNatsPort(value.trim()); - } else if (name == "nats_ws_port") { - setNatsWebsocketPort(value.trim()); - } else { - throw Error("bug"); - } - } -} diff --git a/src/packages/server/nats/index.ts b/src/packages/server/nats/index.ts deleted file mode 100644 index ca3ea7af4c..0000000000 --- a/src/packages/server/nats/index.ts +++ /dev/null @@ -1,39 +0,0 @@ -import getLogger from "@cocalc/backend/logger"; -import { initAPI } from "./api"; -import { init as initDatabase } from "@cocalc/database/nats/changefeeds"; -import { init as initChangefeedServer } from "@cocalc/database/nats/changefeed-api"; -import { init as initLLM } from "./llm"; -import { init as initAuth } from "./auth"; -import { init as initTieredStorage } from "./tiered-storage/api"; -import { loadNatsConfiguration } from "./configuration"; -import { createTimeService } from "@cocalc/nats/service/time"; - -export { loadNatsConfiguration }; - -const logger = getLogger("server:nats"); - -export async function initNatsDatabaseServer() { - await loadNatsConfiguration(); - // do NOT await initDatabase - initDatabase(); -} - -export async function initNatsChangefeedServer() { - await loadNatsConfiguration(); - // do NOT await initDatabase - initChangefeedServer(); -} - -export async function initNatsTieredStorage() { - await loadNatsConfiguration(); - initTieredStorage(); -} - -export async function initNatsServer() { - logger.debug("initializing nats cocalc hub server"); - await loadNatsConfiguration(); - initAPI(); - await initAuth(); - await initLLM(); - createTimeService(); -} diff --git a/src/packages/server/nats/system.ts b/src/packages/server/nats/system.ts deleted file mode 100644 index d23fdcb70f..0000000000 --- a/src/packages/server/nats/system.ts +++ /dev/null @@ -1,22 +0,0 @@ -/* -This seems like it will be really useful... but we're not -using it yet. -*/ - -import { SystemKv } from "@cocalc/nats/system"; -import { JSONCodec } from "nats"; -import { getConnection } from "@cocalc/backend/nats"; -import { sha1 } from "@cocalc/backend/misc_node"; -import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; - -let cache: SystemKv | null = null; -export const systemKv = reuseInFlight(async () => { - if (cache != null) { - return cache; - } - const jc = JSONCodec(); - const nc = await getConnection(); - cache = new SystemKv({ jc, nc, sha1 }); - await cache.init(); - return cache; -}); diff --git a/src/packages/server/nats/tiered-storage/api.ts b/src/packages/server/nats/tiered-storage/api.ts deleted file mode 100644 index 8b4b93f25b..0000000000 --- a/src/packages/server/nats/tiered-storage/api.ts +++ /dev/null @@ -1,128 +0,0 @@ -/* -Our tiered storage code as a server on the nats network. - -DEVELOPMENT: - -If you're running a dev server, turn off the tiered storage service running in it by sending this message from a browser as an admin: - - await cc.client.nats_client.hub.system.terminate({service:'tiered-storage'}) - - -To start this service: - -> a = require('@cocalc/server/nats/tiered-storage'); a.init() - -or - - echo "require('@cocalc/server/nats/tiered-storage').init()" | node - - -To *USE* this service in another terminal: - -> require('@cocalc/backend/nats'); c = require('@cocalc/nats/tiered-storage/client') -{ - state: [AsyncFunction: state], - restore: [AsyncFunction: restore], - archive: [AsyncFunction: archive], - backup: [AsyncFunction: backup], - info: [AsyncFunction: info] -} -> await c.info({project_id:'27cf0030-a9c8-4168-bc03-d0efb3d2269e'}) -{ - subject: 'tiered-storage.project-27cf0030-a9c8-4168-bc03-d0efb3d2269e.api' -} -*/ - -import { - type TieredStorage as TieredStorageInterface, - type Info, - init as initServer, - terminate, -} from "@cocalc/nats/tiered-storage/server"; -import { type Location } from "@cocalc/nats/types"; -import { type LocationType } from "./types"; -import { backupProject, backupAccount } from "./backup"; -import { restoreProject, restoreAccount } from "./restore"; -import { archiveProject, archiveAccount } from "./archive"; -import { getProjectInfo, getAccountInfo } from "./info"; -import { isValidUUID } from "@cocalc/util/misc"; -import "@cocalc/backend/nats"; -import getLogger from "@cocalc/backend/logger"; - -const logger = getLogger("tiered-storage:api"); - -export { terminate }; - -export async function init() { - logger.debug("init"); - const ts = new TieredStorage(); - initServer(ts); -} - -function getType({ account_id, project_id }: Location): LocationType { - if (account_id) { - if (project_id) { - throw Error( - "exactly one of account_id or project_id may be specified but both are", - ); - } - if (!isValidUUID(account_id)) { - throw Error("account_id must be a valid uuid"); - } - return "account"; - } else if (project_id) { - if (!isValidUUID(project_id)) { - throw Error("project_id must be a valid uuid"); - } - return "project"; - } else { - throw Error( - "exactly one of account_id or project_id may be specified but neither are", - ); - } -} - -class TieredStorage implements TieredStorageInterface { - info = async (location: Location): Promise => { - const type = getType(location); - if (type == "account") { - return await getAccountInfo(location as { account_id: string }); - } else if (type == "project") { - return await getProjectInfo(location as { project_id: string }); - } - throw Error("invalid type"); - }; - - restore = async (location: Location): Promise => { - const type = getType(location); - if (type == "account") { - return await restoreAccount(location as { account_id: string }); - } else if (type == "project") { - return await restoreProject(location as { project_id: string }); - } - throw Error("invalid type"); - }; - - archive = async (location: Location): Promise => { - const type = getType(location); - if (type == "account") { - return await archiveAccount(location as { account_id: string }); - } else if (type == "project") { - return await archiveProject(location as { project_id: string }); - } - throw Error("invalid type"); - }; - - backup = async (location: Location): Promise => { - const type = getType(location); - if (type == "account") { - return await backupAccount(location as { account_id: string }); - } else if (type == "project") { - return await backupProject(location as { project_id: string }); - } - throw Error("invalid type"); - }; - - // shut this server down (no-op right now) - close = async (): Promise => {}; -} diff --git a/src/packages/server/nats/tiered-storage/archive.ts b/src/packages/server/nats/tiered-storage/archive.ts deleted file mode 100644 index 9c9caca019..0000000000 --- a/src/packages/server/nats/tiered-storage/archive.ts +++ /dev/null @@ -1,87 +0,0 @@ -import { executeCode } from "@cocalc/backend/execute-code"; -import { natsCoCalcUserEnv } from "@cocalc/backend/nats/cli"; -import { backupStream, backupKV, backupLocation } from "./backup"; -import { restoreKV } from "./restore"; -import type { LocationType } from "./types"; -import getLogger from "@cocalc/backend/logger"; - -const logger = getLogger("tiered-storage:archive"); - -export async function rmStream(name: string) { - // TODO: probably this should be done via the API - const { exit_code, stderr } = await executeCode({ - command: "nats", - args: ["stream", "rm", "-f", name], - err_on_exit: false, - env: natsCoCalcUserEnv(), - }); - if (exit_code) { - if (stderr.includes("stream not found")) { - return; - } - throw Error(stderr); - } -} - -export async function archiveStream(name: string) { - logger.debug("archive", { name }); - const output = await backupStream(name); - await rmStream(name); - return output; -} - -export async function rmKV(name: string) { - // TODO: probably this should be done via the API - const { exit_code, stderr } = await executeCode({ - command: "nats", - args: ["kv", "del", "-f", name], - err_on_exit: false, - env: natsCoCalcUserEnv(), - }); - if (exit_code) { - if (stderr.includes("bucket not found")) { - return; - } - throw Error(stderr); - } -} - -export async function archiveKV(name: string) { - const output = await backupKV(name); - await rmKV(name); - return output; -} - -export async function archiveLocation({ - user_id, - type, -}: { - user_id: string; - type: LocationType; -}) { - const output = await backupLocation({ user_id, type }); - const name = `${type}-${user_id}`; - await rmKV(name); - try { - await rmStream(name); - } catch (err) { - // try to roll back to valid state: - logger.debug( - `unexpected error archiving -- attempting roll back -- ${err} `, - { - name, - }, - ); - await restoreKV(name); - throw err; - } - return output; -} - -export async function archiveProject({ project_id }: { project_id: string }) { - return await archiveLocation({ user_id: project_id, type: "project" }); -} - -export async function archiveAccount({ account_id }: { account_id: string }) { - return await archiveLocation({ user_id: account_id, type: "account" }); -} diff --git a/src/packages/server/nats/tiered-storage/backup.ts b/src/packages/server/nats/tiered-storage/backup.ts deleted file mode 100644 index 75c5eabd1b..0000000000 --- a/src/packages/server/nats/tiered-storage/backup.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { executeCode } from "@cocalc/backend/execute-code"; -import { natsBackup } from "@cocalc/backend/data"; -import { join } from "path"; -import mkdirp from "mkdirp"; -import { natsCoCalcUserEnv } from "@cocalc/backend/nats/cli"; -import getLogger from "@cocalc/backend/logger"; - -const logger = getLogger("tiered-storage:backup"); - -export async function backupStream(name: string) { - logger.debug("backup stream", { name }); - await mkdirp(join(natsBackup, name)); - const { stdout, stderr, exit_code } = await executeCode({ - command: "nats", - args: [ - "stream", - "backup", - "--no-progress", - "--no-consumers", - name, - join(natsBackup, name), - ], - err_on_exit: false, - env: { ...process.env, ...natsCoCalcUserEnv() }, - }); - if (exit_code) { - if (stderr.includes("stream not found")) { - return; - } else { - throw Error(stderr); - } - } - return `${stdout}\n${stderr}`; -} - -export async function backupKV(name: string) { - return await backupStream(`KV_${name}`); -} - -export async function backupLocation({ - user_id, - type, -}: { - user_id: string; - type: "account" | "project"; -}) { - const name = `${type}-${user_id}`; - await backupKV(name); - await backupStream(name); -} - -export async function backupProject({ project_id }: { project_id: string }) { - return await backupLocation({ user_id: project_id, type: "project" }); -} - -export async function backupAccount({ account_id }: { account_id: string }) { - return await backupLocation({ user_id: account_id, type: "account" }); -} diff --git a/src/packages/server/nats/tiered-storage/clean.ts b/src/packages/server/nats/tiered-storage/clean.ts deleted file mode 100644 index 1e50f2ec18..0000000000 --- a/src/packages/server/nats/tiered-storage/clean.ts +++ /dev/null @@ -1,141 +0,0 @@ -/* -Archive inactive things to save on resources. -*/ - -import { getKvManager, getStreamManager } from "./info"; -import "@cocalc/backend/nats"; -import { isValidUUID } from "@cocalc/util/misc"; -import getLogger from "@cocalc/backend/logger"; -import { archiveProject, archiveAccount } from "./archive"; - -const logger = getLogger("tiered-storage:clean"); -const log = (...args) => { - logger.debug(...args); - console.log("tiered-storage:clean: ", ...args); -}; - -const DAY = 1000 * 60 * 60 * 24; - -const DEFAULT_DAYS = 7; -const MIN_DAYS = 3; - -function ageToTimestamp(days: number) { - return Date.now() - days * DAY; -} - -function isProjectOrAccount(name) { - if (!(name.startsWith("account-") || name.startsWith("project-"))) { - return false; - } - if (!isValidUUID(name.slice(-36))) { - return false; - } - return true; -} - -export async function getOldKvs({ - days = DEFAULT_DAYS, -}: { - days?: number; -} = {}) { - const cutoff = ageToTimestamp(days); - const kvm = await getKvManager(); - const names: string[] = []; - for await (const { si } of kvm.list()) { - if (!si.config.name.startsWith("KV_")) { - continue; - } - const name = si.config.name.slice("KV_".length); - if (!isProjectOrAccount(name)) { - continue; - } - const { last_ts } = si.state; - const last = last_ts.startsWith("0001") ? 0 : new Date(last_ts).valueOf(); - if (last <= cutoff) { - names.push(name); - } - } - return names; -} - -export async function getOldStreams({ - days = DEFAULT_DAYS, -}: { - days?: number; -} = {}) { - const cutoff = ageToTimestamp(days); - const jsm = await getStreamManager(); - const names: string[] = []; - for await (const si of jsm.streams.list()) { - const name = si.config.name; - if (!isProjectOrAccount(name)) { - continue; - } - if (name.startsWith("KV_")) { - continue; - } - const { last_ts } = si.state; - const last = last_ts.startsWith("0001") ? 0 : new Date(last_ts).valueOf(); - if (last <= cutoff) { - names.push(name); - } - } - return names; -} - -export async function getOldProjectsAndAccounts({ - days = DEFAULT_DAYS, -}: { - days?: number; -} = {}) { - const kvs = await getOldKvs({ days }); - const streams = await getOldStreams({ days }); - const projects = new Set(); - const accounts = new Set(); - for (const kv of kvs.concat(streams)) { - if (kv.startsWith("account")) { - accounts.add(kv.slice("account-".length)); - } - if (kv.startsWith("project")) { - projects.add(kv.slice("project-".length)); - } - } - return { - accounts: Array.from(accounts).sort(), - projects: Array.from(projects).sort(), - }; -} - -export async function archiveInactive({ - days = DEFAULT_DAYS, - force = false, - dryRun = true, -}: { - days?: number; - force?: boolean; - dryRun?: boolean; -} = {}) { - log("archiveInactive", { days, force, dryRun }); - // step 1 -- get all streams and kv in nats - if (days < MIN_DAYS && !force) { - throw Error(`days is < ${MIN_DAYS} day, which is very suspicious!`); - } - - const { accounts, projects } = await getOldProjectsAndAccounts({ days }); - log( - `archiveInactive: got ${accounts.length} accounts and ${projects.length} projects`, - ); - if (dryRun) { - log(`archiveInactive: dry run so not doing`); - return; - } - - for (const account_id of accounts) { - log(`archiving account ${account_id}`); - await archiveAccount({ account_id }); - } - for (const project_id of projects) { - log(`archiving project ${project_id}`); - await archiveProject({ project_id }); - } -} diff --git a/src/packages/server/nats/tiered-storage/index.ts b/src/packages/server/nats/tiered-storage/index.ts deleted file mode 100644 index bfcd008f1d..0000000000 --- a/src/packages/server/nats/tiered-storage/index.ts +++ /dev/null @@ -1,15 +0,0 @@ -export { init } from "./api"; - -export { backupStream, backupKV, backupProject, backupAccount } from "./backup"; -export { - restoreStream, - restoreKV, - restoreProject, - restoreAccount, -} from "./restore"; -export { - archiveStream, - archiveKV, - archiveProject, - archiveAccount, -} from "./archive"; diff --git a/src/packages/server/nats/tiered-storage/info.ts b/src/packages/server/nats/tiered-storage/info.ts deleted file mode 100644 index 6d1d33ef60..0000000000 --- a/src/packages/server/nats/tiered-storage/info.ts +++ /dev/null @@ -1,98 +0,0 @@ -import { type Info } from "@cocalc/nats/tiered-storage/server"; -import { - jetstreamManager, - type JetStreamManager, - type StreamInfo, -} from "@nats-io/jetstream"; -import { Kvm } from "@nats-io/kv"; -import { getConnection } from "@cocalc/nats/client"; -import { natsBackup } from "@cocalc/backend/data"; -import { join } from "path"; -import { readFile } from "fs/promises"; - -let jsm: null | JetStreamManager = null; -export async function getStreamManager(): Promise { - if (jsm == null) { - jsm = await jetstreamManager(await getConnection()); - } - return jsm; -} - -export async function getNatsStreamInfo(stream: string): Promise { - const jsm = await getStreamManager(); - try { - return await jsm.streams.info(stream); - } catch (err) { - if (err.status == 404) { - // the stream simply doesn't exist -- not just some weird problem contacting the api server - return null; - } - throw err; - } -} - -let kvm: null | Kvm = null; -export async function getKvManager(): Promise { - if (kvm == null) { - kvm = new Kvm(await getConnection()); - } - return kvm; -} - -export async function getNatsKvInfo(bucket: string): Promise { - const kvm = await getKvManager(); - try { - const kv = await kvm.open(bucket); - const status = await kv.status(); - // @ts-ignore - return status.si; - } catch (err) { - if (err.status == 404) { - // the kv simply doesn't exist -- *not* just some weird problem contacting the api server - return null; - } - throw err; - } -} - -async function getBackupInfo(name: string) { - const path = join(natsBackup, name, "backup.json"); - try { - const content = await readFile(path); - return JSON.parse(content.toString()); - } catch (err) { - if (err.code == "ENOENT") { - return null; - } - throw err; - } -} - -async function getInfo({ type, user_id }) { - return { - nats: { - stream: await getNatsStreamInfo(`${type}-${user_id}`), - kv: await getNatsKvInfo(`${type}-${user_id}`), - }, - backup: { - stream: await getBackupInfo(`${type}-${user_id}`), - kv: await getBackupInfo(`KV_${type}-${user_id}`), - }, - }; -} - -export async function getProjectInfo({ project_id }): Promise { - const info = await getInfo({ type: "project", user_id: project_id }); - return { - location: { project_id }, - ...info, - }; -} - -export async function getAccountInfo({ account_id }): Promise { - const info = await getInfo({ type: "account", user_id: account_id }); - return { - location: { account_id }, - ...info, - }; -} diff --git a/src/packages/server/nats/tiered-storage/restore.ts b/src/packages/server/nats/tiered-storage/restore.ts deleted file mode 100644 index 6e1387df64..0000000000 --- a/src/packages/server/nats/tiered-storage/restore.ts +++ /dev/null @@ -1,90 +0,0 @@ -import { executeCode } from "@cocalc/backend/execute-code"; -import { natsBackup } from "@cocalc/backend/data"; -import { join } from "path"; -import { natsCoCalcUserEnv } from "@cocalc/backend/nats/cli"; -import { rmKV, rmStream } from "./archive"; -import { exists } from "@cocalc/backend/misc/async-utils-node"; -import type { LocationType } from "./types"; -import { getNatsStreamInfo, getNatsKvInfo } from "./info"; -import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; -import getLogger from "@cocalc/backend/logger"; -const logger = getLogger("tiered-storage:restore"); - -export const restoreStream = reuseInFlight(async (name: string) => { - if (!(await exists(join(natsBackup, name, "backup.json")))) { - // no data about this stream - non-fatal, since this is how - // we backup never-created streams... and what else are we - // going to do? - logger.debug("restoreStream", { name }, " no backup data"); - return; - } - if (await getNatsStreamInfo(name)) { - // stream already exists in nats - logger.debug("restoreStream", { name }, " stream already exists in nats"); - return; - } - logger.debug("restoreStream", { name }, " restoring from backup"); - const { stdout, stderr } = await executeCode({ - command: "nats", - args: ["stream", "restore", "--no-progress", join(natsBackup, name)], - err_on_exit: true, - env: natsCoCalcUserEnv(), - }); - return `${stderr}\n${stdout}`; -}); - -export const restoreKV = reuseInFlight(async (name: string) => { - if (await getNatsKvInfo(name)) { - // kv already exists in nats - return; - } - return await restoreStream(`KV_${name}`); -}); - -export const restoreLocation = reuseInFlight( - async ({ - user_id, - type, - force, - }: { - user_id: string; - type: LocationType; - force?: boolean; - }) => { - const name = `${type}-${user_id}`; - if (force) { - try { - await rmKV(name); - } catch (err) { - console.log(`${err}`); - } - try { - await rmStream(name); - } catch (err) { - console.log(`${err}`); - } - } - await restoreKV(name); - await restoreStream(name); - }, -); - -export async function restoreProject({ - project_id, - force, -}: { - project_id: string; - force?: boolean; -}) { - return await restoreLocation({ user_id: project_id, type: "project", force }); -} - -export async function restoreAccount({ - account_id, - force, -}: { - account_id: string; - force?: boolean; -}) { - return await restoreLocation({ user_id: account_id, type: "account", force }); -} diff --git a/src/packages/server/nats/tiered-storage/types.ts b/src/packages/server/nats/tiered-storage/types.ts deleted file mode 100644 index 6dbb1b96ac..0000000000 --- a/src/packages/server/nats/tiered-storage/types.ts +++ /dev/null @@ -1 +0,0 @@ -export type LocationType = "project" | "account"; diff --git a/src/packages/server/package.json b/src/packages/server/package.json index 8797aa9862..9c6f6ee9c3 100644 --- a/src/packages/server/package.json +++ b/src/packages/server/package.json @@ -12,10 +12,10 @@ "./compute/maintenance": "./dist/compute/maintenance/index.js", "./database/*": "./dist/database/*.js", "./mentions/*": "./dist/mentions/*.js", - "./nats": "./dist/nats/index.js", - "./nats/api": "./dist/nats/api/index.js", - "./nats/auth": "./dist/nats/auth/index.js", - "./nats/tiered-storage": "./dist/nats/tiered-storage/index.js", + "./conat": "./dist/conat/index.js", + "./conat/api": "./dist/conat/api/index.js", + "./conat/auth": "./dist/conat/auth/index.js", + "./conat/socketio": "./dist/conat/socketio/index.js", "./purchases/*": "./dist/purchases/*.js", "./stripe/*": "./dist/stripe/*.js", "./licenses/purchase": "./dist/licenses/purchase/index.js", @@ -26,53 +26,47 @@ "./settings": "./dist/settings/index.js", "./settings/*": "./dist/settings/*.js" }, - "keywords": [ - "utilities", - "cocalc" - ], + "keywords": ["utilities", "cocalc"], "scripts": { "preinstall": "npx only-allow pnpm", "clean": "rm -rf node_modules dist", "build": "../node_modules/.bin/tsc --build", "tsc": "../node_modules/.bin/tsc --watch --pretty --preserveWatchOutput ", "test": "TZ=UTC jest --forceExit --runInBand", - "prepublishOnly": "test" + "depcheck": "pnpx depcheck", + "prepublishOnly": "test", + "conat-server": "node ./dist/conat/socketio/cluster.js" }, "author": "SageMath, Inc.", "license": "SEE LICENSE.md", "dependencies": { "@cocalc/backend": "workspace:*", + "@cocalc/conat": "workspace:*", "@cocalc/database": "workspace:*", "@cocalc/gcloud-pricing-calculator": "^1.17.0", - "@cocalc/nats": "workspace:*", "@cocalc/server": "workspace:*", "@cocalc/util": "workspace:*", - "@google-ai/generativelanguage": "^3.1.0", "@google-cloud/bigquery": "^7.8.0", "@google-cloud/compute": "^4.7.0", "@google-cloud/monitoring": "^4.1.0", "@google-cloud/storage": "^7.11.1", "@google-cloud/storage-transfer": "^3.3.0", "@google/generative-ai": "^0.14.0", - "@isaacs/ttlcache": "^1.2.1", + "@isaacs/ttlcache": "^1.4.1", "@langchain/anthropic": "^0.3.18", "@langchain/core": "^0.3.46", "@langchain/google-genai": "^0.2.4", "@langchain/mistralai": "^0.2.0", "@langchain/ollama": "^0.2.0", "@langchain/openai": "^0.5.5", - "@nats-io/jetstream": "3.0.0", - "@nats-io/jwt": "0.0.10-5", - "@nats-io/kv": "3.0.0", - "@nats-io/nats-core": "3.0.0", - "@nats-io/nkeys": "^2.0.3", - "@nats-io/services": "3.0.0", "@node-saml/passport-saml": "^5.0.1", "@passport-js/passport-twitter": "^1.0.8", "@passport-next/passport-google-oauth2": "^1.0.0", "@passport-next/passport-oauth2": "^2.1.4", "@sendgrid/client": "^8.1.4", "@sendgrid/mail": "^8.1.4", + "@socket.io/cluster-adapter": "^0.2.2", + "@socket.io/sticky": "^1.0.4", "@zxcvbn-ts/core": "^3.0.4", "@zxcvbn-ts/language-common": "^3.0.4", "@zxcvbn-ts/language-en": "^3.0.2", @@ -82,11 +76,12 @@ "axios": "^1.7.5", "base62": "^2.0.1", "base64-js": "^1.5.1", - "bottleneck": "^2.19.5", "cloudflare": "^2.9.1", + "cookie": "^1.0.0", "cookies": "^0.8.0", "dayjs": "^1.11.11", "dot-object": "^2.1.5", + "express": "^4.21.2", "express-session": "^1.18.1", "google-auth-library": "^9.4.1", "googleapis": "^137.1.0", @@ -98,10 +93,8 @@ "lodash": "^4.17.21", "lru-cache": "^7.18.3", "markdown-it": "^13.0.1", - "mkdirp": "^1.0.4", "ms": "2.1.2", "nanoid": "^3.3.8", - "nats": "^2.29.3", "node-zendesk": "^5.0.13", "nodemailer": "^6.9.16", "openai": "^4.95.1", @@ -114,7 +107,6 @@ "passport-github2": "^0.1.12", "passport-gitlab2": "^5.0.0", "passport-google-oauth20": "^2.0.0", - "passport-ldapauth": "^3.0.1", "passport-oauth": "^1.0.0", "passport-openidconnect": "^0.1.1", "passport-orcid": "^0.0.4", @@ -135,6 +127,7 @@ "@types/dot-object": "^2.1.6", "@types/express": "^4.17.21", "@types/express-session": "^1.18.0", + "@types/jest": "^29.5.14", "@types/lodash": "^4.14.202", "@types/ms": "^0.7.31", "@types/node": "^18.16.14", diff --git a/src/packages/server/projects/collaborators.ts b/src/packages/server/projects/collaborators.ts new file mode 100644 index 0000000000..d441da19eb --- /dev/null +++ b/src/packages/server/projects/collaborators.ts @@ -0,0 +1,327 @@ +/* +Add, remove and invite collaborators on projects. +*/ + +import { db } from "@cocalc/database"; +import { callback2 } from "@cocalc/util/async-utils"; +import isCollaborator from "@cocalc/server/projects/is-collaborator"; +import type { AddCollaborator } from "@cocalc/conat/hub/api/projects"; +import { add_collaborators_to_projects } from "./collab"; +import { + days_ago, + is_array, + is_valid_email_address, + lower_email_address, +} from "@cocalc/util/misc"; +import getLogger from "@cocalc/backend/logger"; +import { send_invite_email } from "@cocalc/server/hub/email"; +import getEmailAddress from "@cocalc/server/accounts/get-email-address"; +import { is_paying_customer } from "@cocalc/database/postgres/account-queries"; +import { project_has_network_access } from "@cocalc/database/postgres/project-queries"; +import { RESEND_INVITE_INTERVAL_DAYS } from "@cocalc/util/consts/invites"; + +const logger = getLogger("project:collaborators"); + +export async function removeCollaborator({ + account_id, + opts, +}: { + account_id: string; + opts: { + account_id; + project_id; + }; +}): Promise { + if (!(await isCollaborator({ account_id, project_id: opts.project_id }))) { + throw Error("user must be a collaborator"); + } + // @ts-ignore + await callback2(db().remove_collaborator_from_project, opts); +} + +export async function addCollaborator({ + account_id, + opts, +}: { + account_id: string; + opts: AddCollaborator; +}): Promise<{ project_id?: string | string[] }> { + let projects: undefined | string | string[] = opts.project_id; + let accounts: undefined | string | string[] = opts.account_id; + let tokens: undefined | string | string[] = opts.token_id; + let is_single_token = false; + + if (tokens) { + if (!is_array(tokens)) { + is_single_token = true; + tokens = [tokens]; + } + // projects will get mutated below as tokens are used + projects = Array(tokens.length).fill(""); + } + if (!is_array(projects)) { + projects = [projects] as string[]; + } + if (!is_array(accounts)) { + accounts = [accounts]; + } + + await add_collaborators_to_projects( + db(), + account_id, + accounts as string[], + projects as string[], + tokens as string[] | undefined, + ); + // Tokens determine the projects, and it may be useful to the client to know what + // project they just got added to! + let project_id; + if (is_single_token) { + project_id = projects[0]; + } else { + project_id = projects; + } + return project_id; +} + +async function allowUrlsInEmails({ + project_id, + account_id, +}: { + project_id: string; + account_id: string; +}) { + return ( + (await is_paying_customer(db(), account_id)) || + (await project_has_network_access(db(), project_id)) + ); +} + +export async function inviteCollaborator({ + account_id, + opts, +}: { + account_id: string; + opts: { + project_id: string; + account_id: string; + title?: string; + link2proj?: string; + replyto?: string; + replyto_name?: string; + email?: string; + subject?: string; + }; +}): Promise { + if (!(await isCollaborator({ account_id, project_id: opts.project_id }))) { + throw Error("user must be a collaborator"); + } + const dbg = (...args) => logger.debug("inviteCollaborator", ...args); + const database = db(); + + // Actually add user to project + await callback2(database.add_user_to_project, { + project_id: opts.project_id, + account_id: opts.account_id, + group: "collaborator", // in future: "invite_collaborator" + }); + + // Everything else in this big function is about notifying the user that they + // were added. + if (!opts.email) { + return; + } + + const email_address = await getEmailAddress(opts.account_id); + if (!email_address) { + return; + } + const when_sent = await callback2(database.when_sent_project_invite, { + project_id: opts.project_id, + to: email_address, + }); + if (when_sent && when_sent >= days_ago(RESEND_INVITE_INTERVAL_DAYS)) { + return; + } + const settings = await callback2(database.get_server_settings_cached); + if (!settings) { + return; + } + dbg(`send_email invite to ${email_address}`); + let subject: string; + if (opts.subject) { + subject = opts.subject; + } else if (opts.replyto_name) { + subject = `${opts.replyto_name} invited you to collaborate on the project '${opts.title}'`; + } else { + subject = `Invitation for collaborating in the project '${opts.title}'`; + } + + try { + await callback2(send_invite_email, { + to: email_address, + subject, + email: opts.email, + email_address, + title: opts.title, + allow_urls: await allowUrlsInEmails({ + account_id, + project_id: opts.project_id, + }), + replyto: opts.replyto ?? settings.organization_email, + replyto_name: opts.replyto_name, + link2proj: opts.link2proj, + settings, + }); + } catch (err) { + dbg(`FAILED to send email to ${email_address} -- ${err}`); + await callback2(database.sent_project_invite, { + project_id: opts.project_id, + to: email_address, + error: `${err}`, + }); + throw err; + } + // Record successful send (without error): + await callback2(database.sent_project_invite, { + project_id: opts.project_id, + to: email_address, + error: undefined, + }); +} + +export async function inviteCollaboratorWithoutAccount({ + account_id, + opts, +}: { + account_id: string; + opts: { + project_id: string; + title: string; + link2proj: string; + replyto?: string; + replyto_name?: string; + to: string; + email: string; // body in HTML format + subject?: string; + }; +}): Promise { + if (!(await isCollaborator({ account_id, project_id: opts.project_id }))) { + throw Error("user must be a collaborator"); + } + const dbg = (...args) => + logger.debug("inviteCollaboratorWithoutAccount", ...args); + const database = db(); + + if (opts.to.length > 1024) { + throw Error( + "Specify less recipients when adding collaborators to project.", + ); + } + + // Prepare list of recipients + const to: string[] = opts.to + .replace(/\s/g, ",") + .replace(/;/g, ",") + .split(",") + .filter((x) => x); + + // Helper for inviting one user by email + const invite_user = async (email_address: string) => { + dbg(`inviting ${email_address}`); + if (!is_valid_email_address(email_address)) { + throw Error(`invalid email address '${email_address}'`); + } + email_address = lower_email_address(email_address); + if (email_address.length >= 128) { + throw Error( + `email address must be at most 128 characters: '${email_address}'`, + ); + } + + // 1. Already have an account? + const to_account_id = await callback2(database.account_exists, { + email_address, + }); + + // 2. If user exists, add to project; otherwise, trigger later add + if (to_account_id) { + dbg(`user ${email_address} already has an account -- add directly`); + await callback2(database.add_user_to_project, { + project_id: opts.project_id, + account_id: to_account_id, + group: "collaborator", + }); + } else { + dbg( + `user ${email_address} doesn't have an account yet -- may send email (if we haven't recently)`, + ); + await callback2(database.account_creation_actions, { + email_address, + action: { + action: "add_to_project", + group: "collaborator", + project_id: opts.project_id, + }, + ttl: 60 * 60 * 24 * 14, + }); + } + + // 3. Has email been sent recently? + const when_sent = await callback2(database.when_sent_project_invite, { + project_id: opts.project_id, + to: email_address, + }); + if (when_sent && when_sent >= days_ago(RESEND_INVITE_INTERVAL_DAYS)) { + // recent email -- nothing more to do + return; + } + + // 4. Get settings + const settings = await callback2(database.get_server_settings_cached); + if (!settings) { + return; + } + + // 5. Send email + + // Compose subject + const subject = opts.subject ? opts.subject : "CoCalc Invitation"; + + dbg(`send_email invite to ${email_address}`); + try { + await callback2(send_invite_email, { + to: email_address, + subject, + email: opts.email, + email_address, + title: opts.title, + allow_urls: await allowUrlsInEmails({ + account_id, + project_id: opts.project_id, + }), + replyto: opts.replyto ?? settings.organization_email, + replyto_name: opts.replyto_name, + link2proj: opts.link2proj, + settings, + }); + } catch (err) { + dbg(`FAILED to send email to ${email_address} -- err=${err}`); + await callback2(database.sent_project_invite, { + project_id: opts.project_id, + to: email_address, + error: `${err}`, + }); + throw err; + } + // Record successful send (without error): + await callback2(database.sent_project_invite, { + project_id: opts.project_id, + to: email_address, + error: undefined, + }); + }; + + // If any invite_user throws, its an error + await Promise.all(to.map((email) => invite_user(email))); +} diff --git a/src/packages/server/projects/connection/connect.ts b/src/packages/server/projects/connection/connect.ts index da32a3bf39..3d99eb82a9 100644 --- a/src/packages/server/projects/connection/connect.ts +++ b/src/packages/server/projects/connection/connect.ts @@ -14,7 +14,6 @@ import { delay } from "awaiting"; import { cancelAll } from "./handle-query"; import initialize from "./initialize"; import { callProjectMessage } from "./handle-message"; - import { CoCalcSocket } from "@cocalc/backend/tcp/enable-messaging-protocol"; import { connectToLockedSocket } from "@cocalc/backend/tcp/locked-socket"; diff --git a/src/packages/server/projects/connection/handle-message.ts b/src/packages/server/projects/connection/handle-message.ts index 255e79cd49..91136b1987 100644 --- a/src/packages/server/projects/connection/handle-message.ts +++ b/src/packages/server/projects/connection/handle-message.ts @@ -9,7 +9,6 @@ import getLogger from "@cocalc/backend/logger"; import { TIMEOUT_CALLING_PROJECT } from "@cocalc/util/consts/project"; import { error, pong } from "@cocalc/util/message"; import handleQuery from "./handle-query"; -import handleSyncdoc from "./handle-syncdoc"; import handleVersion from "./handle-version"; const logger = getLogger("project-connection:handle-message"); @@ -62,9 +61,6 @@ export default async function handleMessage({ case "query_cancel": await handleQuery({ project_id, mesg, sendResponse }); return; - case "get_syncdoc_history": - await handleSyncdoc({ project_id, mesg, sendResponse }); - return; case "file_written_to_project": case "file_read_from_project": case "error": diff --git a/src/packages/server/projects/connection/handle-syncdoc.ts b/src/packages/server/projects/connection/handle-syncdoc.ts deleted file mode 100644 index 0fe8f5172d..0000000000 --- a/src/packages/server/projects/connection/handle-syncdoc.ts +++ /dev/null @@ -1,44 +0,0 @@ -/* -Handle a syncdoc history request -*/ - -import { syncdoc_history } from "@cocalc/util/message"; -import { db } from "@cocalc/database"; -import getPool from "@cocalc/database/pool"; - -interface Options { - project_id: string; - mesg; - sendResponse: (any) => void; -} - -export default async function handleSyncdoc({ - project_id, - mesg, - sendResponse, -}: Options) { - const { patches, string_id } = mesg; - // this raises an error if user does not have access - await checkSyncdocAccess(project_id, string_id); - // get the history - const history = await db().syncdoc_history_async(string_id, patches); - sendResponse(syncdoc_history({ history })); -} - -async function checkSyncdocAccess(project_id, string_id): Promise { - if (typeof string_id != "string" && string_id.length == 40) { - throw Error("invalid string_id"); - } - const pool = getPool("long"); // caching is fine since a "no" result isn't cached and a yes result doesn't change. - const { rows } = await pool.query( - "SELECT project_id FROM syncstrings WHERE string_id = $1::CHAR(40)", - [string_id] - ); - if (rows.length == 0) { - throw Error("no such syncdoc"); - } - if (rows[0].project_id != project_id) { - throw Error("project does NOT have access to this syncdoc"); - } - // everything is fine -- nothing more to do -} diff --git a/src/packages/server/projects/control/base.ts b/src/packages/server/projects/control/base.ts index 606b12daa5..1deb0a1076 100644 --- a/src/packages/server/projects/control/base.ts +++ b/src/packages/server/projects/control/base.ts @@ -21,7 +21,7 @@ possible, so it is manageable, especially as we adapt CoCalc to new environments. */ -import { callback2 } from "@cocalc/util/async-utils"; +import { callback2, until } from "@cocalc/util/async-utils"; import { db } from "@cocalc/database"; import { EventEmitter } from "events"; import { isEqual } from "lodash"; @@ -31,7 +31,6 @@ import { ProjectStatus, } from "@cocalc/util/db-schema/projects"; import { Quota, quota } from "@cocalc/util/upgrades/quota"; -import { delay } from "awaiting"; import getLogger from "@cocalc/backend/logger"; import { site_license_hook } from "@cocalc/database/postgres/site-license/hook"; import { getQuotaSiteSettings } from "@cocalc/database/postgres/site-license/quota-site-settings"; @@ -39,6 +38,7 @@ import getPool from "@cocalc/database/pool"; import { closePayAsYouGoPurchases } from "@cocalc/server/purchases/project-quotas"; import { handlePayAsYouGoQuotas } from "./pay-as-you-go"; import { query } from "@cocalc/database/postgres/query"; +import { getProjectSecretToken } from "./secret-token"; export type { CopyOptions }; export type { ProjectState, ProjectStatus }; @@ -148,20 +148,21 @@ export abstract class BaseProject extends EventEmitter { until: () => Promise; maxTime: number; }): Promise { - const { until, maxTime } = opts; - const t0 = Date.now(); - let d = 250; - while (Date.now() - t0 <= maxTime) { - if (await until()) { - logger.debug(`wait ${this.project_id} -- satisfied`); - return; - } - await delay(d); - d *= 1.2; - } - const err = `wait ${this.project_id} -- FAILED`; - logger.debug(err); - throw Error(err); + await until( + async () => { + if (await opts.until()) { + logger.debug(`wait ${this.project_id} -- satisfied`); + return true; + } + return false; + }, + { + start: 250, + decay: 1.25, + max: opts.maxTime, + log: (...args) => logger.debug("wait", this.project_id, ...args), + }, + ); } // Everything the hub needs to know to connect to the project @@ -180,9 +181,6 @@ export abstract class BaseProject extends EventEmitter { if (!status["hub-server.port"]) { throw Error("unable to determine project port"); } - if (!status["secret_token"]) { - throw Error("unable to determine secret_token"); - } const state = await this.state(); const host = state.ip; if (!host) { @@ -191,7 +189,7 @@ export abstract class BaseProject extends EventEmitter { return { host, port: status["hub-server.port"], - secret_token: status.secret_token, + secret_token: await getProjectSecretToken(this.project_id), }; } diff --git a/src/packages/server/projects/control/multi-user.ts b/src/packages/server/projects/control/multi-user.ts index bf39e6617b..f5e377bd2b 100644 --- a/src/packages/server/projects/control/multi-user.ts +++ b/src/packages/server/projects/control/multi-user.ts @@ -31,6 +31,7 @@ import { mkdir, setupDataPath, stopProjectProcesses, + writeSecretToken, } from "./util"; import { BaseProject, @@ -41,6 +42,10 @@ import { } from "./base"; import getLogger from "@cocalc/backend/logger"; import { getUid } from "@cocalc/backend/misc"; +import { + deleteProjectSecretToken, + getProjectSecretToken, +} from "./secret-token"; const winston = getLogger("project-control:multi-user"); @@ -71,7 +76,7 @@ class Project extends BaseProject { const status = await getStatus(this.HOME); // TODO: don't include secret token in log message. winston.debug( - `got status of ${this.project_id} = ${JSON.stringify(status)}` + `got status of ${this.project_id} = ${JSON.stringify(status)}`, ); this.saveStatusToDatabase(status); return status; @@ -107,6 +112,12 @@ class Project extends BaseProject { // Setup files await setupDataPath(HOME, this.uid); + await writeSecretToken( + HOME, + await getProjectSecretToken(this.project_id), + this.uid, + ); + // Fork and launch project server daemon await launchProjectDaemon(env, this.uid); @@ -116,7 +127,7 @@ class Project extends BaseProject { return false; } const status = await this.status(); - return !!status.secret_token && !!status["hub-server.port"]; + return !!status["hub-server.port"]; }, maxTime: MAX_START_TIME_MS, }); @@ -143,6 +154,7 @@ class Project extends BaseProject { until: async () => !(await isProjectRunning(this.HOME)), maxTime: MAX_STOP_TIME_MS, }); + await deleteProjectSecretToken(this.project_id); } finally { this.stateChanging = undefined; // ensure state valid in database @@ -155,7 +167,7 @@ class Project extends BaseProject { await copyPath( opts, this.project_id, - opts.target_project_id ? getUid(opts.target_project_id) : undefined + opts.target_project_id ? getUid(opts.target_project_id) : undefined, ); return ""; } diff --git a/src/packages/server/projects/control/secret-token.ts b/src/packages/server/projects/control/secret-token.ts new file mode 100644 index 0000000000..81677c9ea7 --- /dev/null +++ b/src/packages/server/projects/control/secret-token.ts @@ -0,0 +1,34 @@ +import { secureRandomString } from "@cocalc/backend/misc"; +import getPool from "@cocalc/database/pool"; + +const SECRET_TOKEN_LENGTH = 32; + +export async function getProjectSecretToken(project_id): Promise { + const pool = getPool(); + const { rows } = await pool.query( + "select secret_token from projects where project_id=$1", + [project_id], + ); + if (rows.length == 0) { + throw Error(`no project ${project_id}`); + } + if (!rows[0].secret_token) { + const secret_token = await secureRandomString(SECRET_TOKEN_LENGTH); + await pool.query( + "UPDATE projects SET secret_token=$1 where project_id=$2", + [secret_token, project_id], + ); + return secret_token; + } + return rows[0]?.secret_token; +} + +export async function deleteProjectSecretToken( + project_id, +): Promise { + const pool = getPool(); + await pool.query( + "UPDATE projects SET secret_token=NULL WHERE project_id=$1", + [project_id], + ); +} diff --git a/src/packages/server/projects/control/single-user.ts b/src/packages/server/projects/control/single-user.ts index 7a0fcd8754..0801c8c8cb 100644 --- a/src/packages/server/projects/control/single-user.ts +++ b/src/packages/server/projects/control/single-user.ts @@ -14,6 +14,17 @@ This is useful for: - development of cocalc from inside of a CoCalc project - non-collaborative use of cocalc on your own laptop, e.g., when you're on an airplane. + + +DEVELOPMENT: + + +~/cocalc/src/packages/server/projects/control$ COCALC_MODE='single-user' node +Welcome to Node.js v20.19.1. +Type ".help" for more information. +> a = require('@cocalc/server/projects/control'); +> p = a.getProject('8a840733-93b6-415c-83d4-7e5712a6266b') +> await p.start() */ import { kill } from "process"; @@ -38,7 +49,12 @@ import { launchProjectDaemon, mkdir, setupDataPath, + writeSecretToken, } from "./util"; +import { + getProjectSecretToken, + deleteProjectSecretToken, +} from "./secret-token"; const logger = getLogger("project-control:single-user"); @@ -100,6 +116,11 @@ class Project extends BaseProject { // Setup files await setupDataPath(HOME); + await writeSecretToken( + HOME, + await getProjectSecretToken(this.project_id), + ); + // Fork and launch project server await launchProjectDaemon(env); @@ -109,7 +130,7 @@ class Project extends BaseProject { return false; } const status = await this.status(); - return !!status.secret_token && !!status["hub-server.port"]; + return !!status["hub-server.port"]; }, maxTime: MAX_START_TIME_MS, }); @@ -143,6 +164,7 @@ class Project extends BaseProject { until: async () => !(await isProjectRunning(this.HOME)), maxTime: MAX_STOP_TIME_MS, }); + await deleteProjectSecretToken(this.project_id); logger.debug("stop: project is not running"); } finally { this.stateChanging = undefined; diff --git a/src/packages/server/projects/control/stop-idle-projects.test.ts b/src/packages/server/projects/control/stop-idle-projects.test.ts index a167ce06c2..69926aa51f 100644 --- a/src/packages/server/projects/control/stop-idle-projects.test.ts +++ b/src/packages/server/projects/control/stop-idle-projects.test.ts @@ -8,6 +8,7 @@ import getPool, { initEphemeralDatabase } from "@cocalc/database/pool"; import { isValidUUID } from "@cocalc/util/misc"; import { test } from "./stop-idle-projects"; const { stopIdleProjects } = test; +import { delay } from "awaiting"; beforeAll(async () => { await initEphemeralDatabase(); @@ -45,7 +46,7 @@ describe("creates a project, set various parameters, and runs idle project funct await pool.query( `UPDATE projects SET run_quota='{"network": false, "cpu_limit": 1, "disk_quota": 3000, "privileged": false, "cpu_request": 0.02, "member_host": false, "dedicated_vm": false, "idle_timeout": 1800, "memory_limit": 1000, "always_running": false, "memory_request": 200, "dedicated_disks": []}', last_edited=NOW(), last_started=NOW(), state='{"state":"running"}' WHERE project_id=$1`, - [project_id] + [project_id], ); await stopIdleProjects(stopProject); expect(projectsThatGotStopped.has(project_id)).toBe(false); @@ -54,16 +55,19 @@ describe("creates a project, set various parameters, and runs idle project funct it("changes our project so that last_edited is an hour ago and last_started is an hour ago, and observe project gets stopped", async () => { await pool.query( `UPDATE projects SET last_edited=NOW()-interval '1 hour', last_started=NOW()-interval '1 hour' WHERE project_id=$1`, - [project_id] + [project_id], ); await stopIdleProjects(stopProject); + while (!projectsThatGotStopped.has(project_id)) { + await delay(30); + } expect(projectsThatGotStopped.has(project_id)).toBe(true); }); it("changes our project so that last_edited is an hour ago and last_started is a minute ago, and observe project does NOT get stopped", async () => { await pool.query( `UPDATE projects SET last_edited=NOW()-interval '1 hour', last_started=NOW()-interval '1 minute' WHERE project_id=$1`, - [project_id] + [project_id], ); reset(); await stopIdleProjects(stopProject); @@ -73,7 +77,7 @@ describe("creates a project, set various parameters, and runs idle project funct it("changes our project so that last_edited is a minute ago and last_started is an hour ago, and observe project does NOT get stopped", async () => { await pool.query( `UPDATE projects SET last_edited=NOW()-interval '1 minute', last_started=NOW()-interval '1 hour' WHERE project_id=$1`, - [project_id] + [project_id], ); reset(); await stopIdleProjects(stopProject); @@ -84,7 +88,7 @@ describe("creates a project, set various parameters, and runs idle project funct await pool.query( `UPDATE projects SET run_quota='{"network": false, "cpu_limit": 1, "disk_quota": 3000, "privileged": false, "cpu_request": 0.02, "member_host": false, "dedicated_vm": false, "idle_timeout": 1800, "memory_limit": 1000, "always_running": true, "memory_request": 200, "dedicated_disks": []}', last_edited=NOW()-interval '1 month', last_started=NOW()-interval '1 month' WHERE project_id=$1`, - [project_id] + [project_id], ); reset(); await stopIdleProjects(stopProject); @@ -95,11 +99,14 @@ describe("creates a project, set various parameters, and runs idle project funct await pool.query( `UPDATE projects SET run_quota='{"network": false, "cpu_limit": 1, "disk_quota": 3000, "privileged": false, "cpu_request": 0.02, "member_host": false, "dedicated_vm": false, "idle_timeout": 1800, "memory_limit": 1000, "always_running": false, "memory_request": 200, "dedicated_disks": []}', last_edited=NOW()-interval '1 month', last_started=NOW()-interval '1 month', state='{"state":"running"}' WHERE project_id=$1`, - [project_id] + [project_id], ); // first confirm stopProject2 will get called reset(); await stopIdleProjects(stopProject); + while (!projectsThatGotStopped.has(project_id)) { + await delay(30); + } expect(projectsThatGotStopped.has(project_id)).toBe(true); // now call again with error but doesn't break anything const stopProject2 = async (project_id) => { @@ -108,6 +115,9 @@ describe("creates a project, set various parameters, and runs idle project funct }; reset(); await stopIdleProjects(stopProject2); + while (!projectsThatGotStopped.has(project_id)) { + await delay(30); + } expect(projectsThatGotStopped.has(project_id)).toBe(true); }); }); diff --git a/src/packages/server/projects/control/util.ts b/src/packages/server/projects/control/util.ts index c88b2b4ccc..574257ef9a 100644 --- a/src/packages/server/projects/control/util.ts +++ b/src/packages/server/projects/control/util.ts @@ -13,10 +13,10 @@ import { getUid } from "@cocalc/backend/misc"; import base_path from "@cocalc/backend/base-path"; import { db } from "@cocalc/database"; import { getProject } from "."; +import { conatServer } from "@cocalc/backend/data"; import { pidFilename } from "@cocalc/util/project-info"; -import { getServerSettings } from "@cocalc/database/settings/server-settings"; -import { natsPorts, natsServer } from "@cocalc/backend/data"; import { executeCode } from "@cocalc/backend/execute-code"; +import ensureContainingDirectoryExists from "@cocalc/backend/misc/ensure-containing-directory-exists"; const logger = getLogger("project-control:util"); @@ -101,6 +101,25 @@ export async function setupDataPath(HOME: string, uid?: number): Promise { } } +// see also packages/project/secret-token.ts +export function secretTokenPath(HOME: string) { + const data = dataPath(HOME); + return join(data, "secret-token"); +} + +export async function writeSecretToken( + HOME: string, + secretToken: string, + uid?: number, +): Promise { + const path = secretTokenPath(HOME); + await ensureContainingDirectoryExists(path); + await writeFile(path, secretToken); + if (uid) { + await chown(path, uid); + } +} + async function logLaunchParams(params): Promise { const data = dataPath(params.env.HOME); const path = join(data, "launch-params.json"); @@ -129,16 +148,33 @@ export async function launchProjectDaemon(env, uid?: number): Promise { uid, gid: uid, }); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (data) => { + stdout += data.toString(); + if (stdout.length > 10000) { + stdout = stdout.slice(-5000); + } + }); + child.stderr.on("data", (data) => { + stderr += data.toString(); + if (stderr.length > 10000) { + stderr = stderr.slice(-5000); + } + }); child.on("error", (err) => { - logger.debug(`project daemon error ${err}`); + logger.debug(`project daemon error ${err} -- \n${stdout}\n${stderr}`); cb(err); }); child.on("exit", async (code) => { - logger.debug("project daemon exited with code", code); + logger.debug("project daemon exited", { code, stdout, stderr }); if (code != 0) { try { const s = (await readFile(env.LOGS)).toString(); - logger.debug("project log file ended: ", s.slice(-2000)); + logger.debug("project log file ended: ", s.slice(-2000), { + stdout, + stderr, + }); } catch (err) { // there's a lot of reasons the log file might not even exist, // e.g., debugging is not enabled @@ -218,6 +254,7 @@ const ENV_VARS_DELETE = [ "LS_COLORS", "INIT_CWD", "DEBUG_FILE", + "SECRETS", ] as const; export function sanitizedEnv(env: { [key: string]: string | undefined }): { @@ -237,6 +274,7 @@ export function sanitizedEnv(env: { [key: string]: string | undefined }): { if ( key.startsWith("npm_") || key.startsWith("COCALC_") || + key.startsWith("CONAT_") || key.startsWith("PNPM_") || key.startsWith("__NEXT") || key.startsWith("NODE_") || @@ -249,19 +287,6 @@ export function sanitizedEnv(env: { [key: string]: string | undefined }): { return env2 as { [key: string]: string }; } -async function natsWebsocketServer() { - const { nats_project_server } = await getServerSettings(); - if (nats_project_server) { - if (nats_project_server.startsWith("ws")) { - if (base_path.length <= 1) { - return nats_project_server; - } - return `${nats_project_server}${base_path}/nats`; - } - } - return `${natsServer}:${natsPorts.server}`; -} - export async function getEnvironment( project_id: string, ): Promise<{ [key: string]: any }> { @@ -283,7 +308,7 @@ export async function getEnvironment( HOME, BASE_PATH: base_path, DATA, - LOGS: join(DATA, "logs"), + LOGS: DATA, DEBUG: "cocalc:*,-cocalc:silly:*", // so interesting stuff gets logged, but not too much unless we really need it. // important to reset the COCALC_ vars since server env has own in a project COCALC_PROJECT_ID: project_id, @@ -291,8 +316,8 @@ export async function getEnvironment( USER, COCALC_EXTRA_ENV: extra_env, PATH: `${HOME}/bin:${HOME}/.local/bin:${process.env.PATH}`, - // url of the NATS websocket server the project will connect to: - NATS_SERVER: await natsWebsocketServer(), + CONAT_SERVER: conatServer, + COCALC_SECRET_TOKEN: secretTokenPath(HOME), }, }; } @@ -327,7 +352,6 @@ export async function getStatus(HOME: string): Promise { "browser-server.port", "sage_server.port", "sage_server.pid", - "secret_token", "start-timestamp.txt", "session-id.txt", ]) { diff --git a/src/packages/server/test/setup.js b/src/packages/server/test/setup.js index ee2e6cce0d..68b402d5f5 100644 --- a/src/packages/server/test/setup.js +++ b/src/packages/server/test/setup.js @@ -5,3 +5,5 @@ process.env.PGDATABASE = "smc_ephemeral_testing_database"; // checked for in some code to behave differently while running unit tests. process.env.COCALC_TEST_MODE = true; + +process.env.COCALC_MODE = "single-user"; diff --git a/src/packages/server/tsconfig.json b/src/packages/server/tsconfig.json index d08ee56187..3fb51ed22e 100644 --- a/src/packages/server/tsconfig.json +++ b/src/packages/server/tsconfig.json @@ -10,7 +10,7 @@ "references": [ { "path": "../backend" }, { "path": "../database" }, - { "path": "../nats" }, + { "path": "../conat" }, { "path": "../util" } ] } diff --git a/src/packages/static/package.json b/src/packages/static/package.json index 93cd28a128..e8372de1e4 100644 --- a/src/packages/static/package.json +++ b/src/packages/static/package.json @@ -51,9 +51,9 @@ "@cocalc/util": "workspace:*" }, "devDependencies": { - "@rspack/cli": "^1.1.1", - "@rspack/core": "^1.1.1", - "@rspack/plugin-react-refresh": "^1.0.0", + "@rspack/cli": "^1.3.11", + "@rspack/core": "^1.3.11", + "@rspack/plugin-react-refresh": "^1.4.3", "@types/jquery": "^3.5.5", "@types/node": "^18.16.14", "@types/react": "^18.3.10", diff --git a/src/packages/static/src/load.tsx b/src/packages/static/src/load.tsx index f8625f26b9..84c9fcb927 100644 --- a/src/packages/static/src/load.tsx +++ b/src/packages/static/src/load.tsx @@ -8,7 +8,6 @@ import Favicons from "./favicons"; import Manifest from "./manifest"; import Meta from "./meta"; import PreflightCheck from "./preflight-checks"; -import Primus from "./primus"; import StartupBanner from "./startup-banner"; import initError from "./webapp-error"; @@ -18,7 +17,6 @@ const loadContainer = document.getElementById("cocalc-load-container"); if (loadContainer) { createRoot(loadContainer).render( <> - diff --git a/src/packages/static/src/manifest.tsx b/src/packages/static/src/manifest.tsx index fd4d306132..3545e5a303 100644 --- a/src/packages/static/src/manifest.tsx +++ b/src/packages/static/src/manifest.tsx @@ -5,8 +5,12 @@ import { Helmet } from "react-helmet"; import { join } from "path"; import { appBasePath } from "@cocalc/frontend/customize/app-base-path"; +declare var DEBUG; window.addEventListener("load", async function () { + if (DEBUG) { + return null; + } const path = join(appBasePath, "webapp/serviceWorker.js"); try { @@ -19,8 +23,6 @@ window.addEventListener("load", async function () { } }); -declare var DEBUG; - export default function Manifest() { if (DEBUG) { return null; diff --git a/src/packages/static/src/preflight-checks.tsx b/src/packages/static/src/preflight-checks.tsx index b399a1a923..62642f8303 100644 --- a/src/packages/static/src/preflight-checks.tsx +++ b/src/packages/static/src/preflight-checks.tsx @@ -103,7 +103,6 @@ export default function PreflightCheck() { }, [allowed]); if (allowed) { - console.log("Browser is supported."); return null; } diff --git a/src/packages/static/src/primus.tsx b/src/packages/static/src/primus.tsx deleted file mode 100644 index 58f270a09c..0000000000 --- a/src/packages/static/src/primus.tsx +++ /dev/null @@ -1,19 +0,0 @@ -// Load the custom manifest for our site, which is necessary so that we can -// install the page as a local webapp. It's part of being a "progressive -// web app", as was started in this PR: https://github.com/sagemathinc/cocalc/pull/5254 - -import { Helmet } from "react-helmet"; -import { join } from "path"; -import { appBasePath } from "@cocalc/frontend/customize/app-base-path"; - -export default function Primus() { - return ( - - - - ); -} diff --git a/src/packages/static/src/webapp-error-reporter.js b/src/packages/static/src/webapp-error-reporter.js index b959302e39..7140fa1864 100644 --- a/src/packages/static/src/webapp-error-reporter.js +++ b/src/packages/static/src/webapp-error-reporter.js @@ -139,7 +139,7 @@ const sendError = async function (opts) { // As an added bonus, by try/catching and retrying once at least, // we are more likely to get the error report in case of a temporary // network or other glitch.... - console.log("sendError: import webapp_client"); + // console.log("sendError: import webapp_client"); ({ webapp_client } = require("@cocalc/frontend/webapp-client")); // can possibly be undefined // console.log 'sendError: sending error' diff --git a/src/packages/sync-client/lib/api.ts b/src/packages/sync-client/lib/api.ts index d3823e74e3..647803ce8c 100644 --- a/src/packages/sync-client/lib/api.ts +++ b/src/packages/sync-client/lib/api.ts @@ -97,44 +97,6 @@ export default class API implements API_Interface { return this.conn.channel(channel_name); } - // Get the sync *channel* for the given SyncTable project query. - async synctable_channel( - query: { [field: string]: any }, - options: { [field: string]: any }[], - ): Promise { - const channel_name = await this.call( - { - cmd: "synctable_channel", - query, - options, - }, - 10000, - ); - // console.log("synctable_channel", query, options, channel_name); - return this.conn.channel(channel_name); - } - - // Command-response API for synctables. - // - mesg = {cmd:'close'} -- closes the synctable, even if persistent. - async syncdoc_call( - path: string, - mesg: { [field: string]: any }, - timeout_ms: number = 30000, // ms timeout for call - ): Promise { - return await this.call({ cmd: "syncdoc_call", path, mesg }, timeout_ms); - } - - async symmetric_channel(name: string) { - const channel_name = await this.call( - { - cmd: "symmetric_channel", - name, - }, - 30000, - ); - return this.conn.channel(channel_name); - } - async query(opts: any): Promise { if (opts.timeout == null) { opts.timeout = 30000; diff --git a/src/packages/sync-client/lib/index.ts b/src/packages/sync-client/lib/index.ts index b050808b98..35d9da7d9a 100644 --- a/src/packages/sync-client/lib/index.ts +++ b/src/packages/sync-client/lib/index.ts @@ -57,9 +57,6 @@ export default class Client extends EventEmitter implements AppClient { this.project_client = bind_methods(new ProjectClient()); this.sync_client = bind_methods(new SyncClient(this)); - this.synctable_project = this.sync_client.synctable_project.bind( - this.sync_client, - ); } client_id = () => { diff --git a/src/packages/sync-client/package.json b/src/packages/sync-client/package.json index d82d0a9056..dc9ded7847 100644 --- a/src/packages/sync-client/package.json +++ b/src/packages/sync-client/package.json @@ -8,23 +8,16 @@ "./*": "./dist/*.js", "./lib/*": "./dist/lib/*.js" }, - "files": [ - "dist/**", - "bin/**", - "README.md", - "package.json" - ], + "files": ["dist/**", "bin/**", "README.md", "package.json"], "scripts": { "preinstall": "npx only-allow pnpm", "build": "../node_modules/.bin/tsc --build", "clean": "rm -rf node_modules dist", - "tsc": "../node_modules/.bin/tsc --watch --pretty --preserveWatchOutput" + "tsc": "../node_modules/.bin/tsc --watch --pretty --preserveWatchOutput", + "depcheck": "pnpx depcheck" }, "author": "SageMath, Inc.", - "keywords": [ - "cocalc", - "jupyter" - ], + "keywords": ["cocalc", "jupyter"], "license": "SEE LICENSE.md", "dependencies": { "@cocalc/api-client": "workspace:*", @@ -46,6 +39,7 @@ "devDependencies": { "@types/cookie": "^0.6.0", "@types/debug": "^4.1.12", + "@types/jest": "^29.5.14", "@types/node": "^18.16.14", "@types/primus": "^7.3.9" } diff --git a/src/packages/sync-fs/README.md b/src/packages/sync-fs/README.md index 8445bee043..eac355bc7f 100644 --- a/src/packages/sync-fs/README.md +++ b/src/packages/sync-fs/README.md @@ -26,7 +26,7 @@ Running 'find' as a subcommand seems optimal, taking a few KB memory and about - TODO: This sync protocol does NOT deal with file permissions, e.g., changing a file to be executable when it wasn't, since that doesn't update the mtime. See https://github.com/sagemathinc/cocalc/issues/7342 - Dependencies: this doesn't depend on @cocalc/project, but you do need to import -say @cocalc/project/nats before using this code, so that the client process knows +say @cocalc/project/conat before using this code, so that the client process knows how to connect to NATS. ## ALGORITHM diff --git a/src/packages/sync-fs/lib/compressed-json.ts b/src/packages/sync-fs/lib/compressed-json.ts index d0a6543c9e..70eb5f4bf3 100644 --- a/src/packages/sync-fs/lib/compressed-json.ts +++ b/src/packages/sync-fs/lib/compressed-json.ts @@ -1,6 +1,14 @@ /* Compress and deocmpression JSON to a Buffer. This buffer *is* suitable to write to an lz4 file and lz4 -d will work with it. + +NOTE: I was worried because lz4-napi's compressSync and uncompressSync +seem to have a MASSIVE memory leak. I tested these functions via the +following, and did NOT observe a memory leak. So it's maybe just a problem +with their sync functions, fortunately. + +a = require('@cocalc/sync-fs/lib/compressed-json') +t=Date.now(); for(i=0;i<10000;i++) { await a.fromCompressedJSON(await a.toCompressedJSON({a:'x'.repeat(1000000)}))}; Date.now()-t */ import { compressFrame, decompressFrame } from "lz4-napi"; diff --git a/src/packages/sync-fs/lib/nats/syncfs-client.ts b/src/packages/sync-fs/lib/conat/syncfs-client.ts similarity index 87% rename from src/packages/sync-fs/lib/nats/syncfs-client.ts rename to src/packages/sync-fs/lib/conat/syncfs-client.ts index 0865a0ca1a..badc313421 100644 --- a/src/packages/sync-fs/lib/nats/syncfs-client.ts +++ b/src/packages/sync-fs/lib/conat/syncfs-client.ts @@ -2,10 +2,10 @@ SyncFS Client Service, which runs on compute servers */ -import { createSyncFsClientService } from "@cocalc/nats/service/syncfs-client"; +import { createSyncFsClientService } from "@cocalc/conat/service/syncfs-client"; import { type SyncFS } from "../index"; -export async function initNatsClientService({ +export async function initConatClientService({ syncfs, compute_server_id, project_id, diff --git a/src/packages/sync-fs/lib/nats/syncfs-server.ts b/src/packages/sync-fs/lib/conat/syncfs-server.ts similarity index 71% rename from src/packages/sync-fs/lib/nats/syncfs-server.ts rename to src/packages/sync-fs/lib/conat/syncfs-server.ts index f4ba57595a..00623c91ec 100644 --- a/src/packages/sync-fs/lib/nats/syncfs-server.ts +++ b/src/packages/sync-fs/lib/conat/syncfs-server.ts @@ -2,10 +2,10 @@ SyncFS Server Service, which runs in the home base. */ -import { createSyncFsServerService } from "@cocalc/nats/service/syncfs-server"; +import { createSyncFsServerService } from "@cocalc/conat/service/syncfs-server"; import { type SyncFS } from "../index"; -export async function initNatsServerService({ +export async function initConatServerService({ syncfs, project_id, }: { diff --git a/src/packages/sync-fs/lib/index.ts b/src/packages/sync-fs/lib/index.ts index f041ffa360..1e6ce3be39 100644 --- a/src/packages/sync-fs/lib/index.ts +++ b/src/packages/sync-fs/lib/index.ts @@ -14,7 +14,7 @@ import { } from "fs/promises"; import { basename, dirname, join } from "path"; import type { FilesystemState /*FilesystemStatePatch*/ } from "./types"; -import { execa, mtimeDirTree, parseCommonPrefixes, remove } from "./util"; +import { exec, mtimeDirTree, parseCommonPrefixes, remove } from "./util"; import { toCompressedJSON } from "./compressed-json"; import SyncClient, { type Role } from "@cocalc/sync-client/lib/index"; import { encodeIntToUUID } from "@cocalc/util/compute/manager"; @@ -28,8 +28,8 @@ import { executeCode } from "@cocalc/backend/execute-code"; import { delete_files } from "@cocalc/backend/files/delete-files"; import { move_files } from "@cocalc/backend/files/move-files"; import { rename_file } from "@cocalc/backend/files/rename-file"; -import { initNatsClientService } from "./nats/syncfs-client"; -import { initNatsServerService } from "./nats/syncfs-server"; +import { initConatClientService } from "./conat/syncfs-client"; +import { initConatServerService } from "./conat/syncfs-server"; const EXPLICIT_HIDDEN_EXCLUDES = [".cache", ".local"]; @@ -96,7 +96,7 @@ export class SyncFS { private tar: { send; get }; // number of failures in a row to sync. private numFails: number = 0; - private natsService; + private conatService; private client: SyncClient; @@ -166,7 +166,7 @@ export class SyncFS { } init = async () => { - await this.initNatsService(); + await this.initConatService(); await this.mountUnionFS(); await this.bindMountExcludes(); await this.makeScratchDir(); @@ -183,9 +183,9 @@ export class SyncFS { return; } this.state = "closed"; - if (this.natsService != null) { - this.natsService.close(); - delete this.natsService; + if (this.conatService != null) { + this.conatService.close(); + delete this.conatService; } if (this.timeout != null) { clearTimeout(this.timeout); @@ -198,7 +198,7 @@ export class SyncFS { const args = ["-uz", this.mount]; log("fusermount", args.join(" ")); try { - await execa("fusermount", args); + await exec("fusermount", args); } catch (err) { log("fusermount fail -- ", err); } @@ -357,7 +357,7 @@ export class SyncFS { // NOTE: allow_other is essential to allow bind mounted as root // of fast scratch directories into HOME! // unionfs-fuse -o allow_other,auto_unmount,nonempty,large_read,cow,max_files=32768 /upper=RW:/home/user=RO /merged - await execa("unionfs-fuse", [ + await exec("unionfs-fuse", [ "-o", "allow_other,auto_unmount,nonempty,large_read,cow,max_files=32768", `${this.upper}=RW:${this.lower}=RO`, @@ -381,7 +381,7 @@ export class SyncFS { try { const target = join(this.mount, path); log("unmountExcludes -- unmounting", { target }); - await execa("sudo", ["umount", target]); + await exec("sudo", ["umount", target]); } catch (err) { log("unmountExcludes -- warning ", err); } @@ -403,7 +403,7 @@ export class SyncFS { // Yes, we have to mkdir in the upper level of the unionfs, because // we excluded this path from the websocketfs metadataFile caching. await mkdirp(upper); - await execa("sudo", ["mount", "--bind", source, target]); + await exec("sudo", ["mount", "--bind", source, target]); } else { log("bindMountExcludes -- skipping", { path }); } @@ -419,7 +419,7 @@ export class SyncFS { log("bindMountExcludes -- explicit hidden path", { source, target }); await mkdirp(source); await mkdirp(upper); - await execa("sudo", ["mount", "--bind", source, target]); + await exec("sudo", ["mount", "--bind", source, target]); } }; @@ -798,15 +798,15 @@ export class SyncFS { } }; - initNatsService = async () => { + initConatService = async () => { if (this.role == "compute_server") { - this.natsService = await initNatsClientService({ + this.conatService = await initConatClientService({ syncfs: this, project_id: this.project_id, compute_server_id: this.compute_server_id, }); } else if (this.role == "project") { - this.natsService = await initNatsServerService({ + this.conatService = await initConatServerService({ syncfs: this, project_id: this.project_id, }); diff --git a/src/packages/sync-fs/lib/util.ts b/src/packages/sync-fs/lib/util.ts index 709490b5ea..4ba38b94d9 100644 --- a/src/packages/sync-fs/lib/util.ts +++ b/src/packages/sync-fs/lib/util.ts @@ -1,19 +1,15 @@ -import { dynamicImport } from "tsimportlib"; import { readdir, rm, writeFile } from "fs/promises"; import { dirname, join } from "path"; import { exists } from "@cocalc/backend/misc/async-utils-node"; import { compressFrame } from "lz4-napi"; +import { executeCode } from "@cocalc/backend/execute-code"; import getLogger from "@cocalc/backend/logger"; const log = getLogger("sync-fs:util").debug; -export async function execa(cmd, args, options?) { - log("execa", cmd, "...", args?.slice(-15)?.join(" "), options); - const { execa: execa0 } = (await dynamicImport( - "execa", - module, - )) as typeof import("execa"); - return await execa0(cmd, args, options); +export async function exec(command: string, args?: string[], options?) { + log("exec", command, "...", args?.slice(-15)?.join(" "), options); + return await executeCode({ command, args, ...options }); } // IMPORTANT: top level hidden subdirectories in path are always ignored, e.g., @@ -47,7 +43,7 @@ export async function metadataFile({ const topPaths = (await readdir(path)).filter( (p) => !p.startsWith(".") && !exclude.includes(p), ); - const { stdout } = await execa( + const { stdout } = await exec( "find", topPaths.concat([ // This '-not -readable -prune -o ' excludes directories that we can read, since there is no possible @@ -102,7 +98,7 @@ export async function mtimeDirTree({ "-printf", "%p\\0%T@\\0\\0", ]); - const { stdout } = await execa("find", [...args], { + const { stdout } = await exec("find", [...args], { cwd: path, }); metadataFile = stdout; diff --git a/src/packages/sync-fs/package.json b/src/packages/sync-fs/package.json index c0bed88cf4..08b34f4b7c 100644 --- a/src/packages/sync-fs/package.json +++ b/src/packages/sync-fs/package.json @@ -8,29 +8,36 @@ "./*": "./dist/*.js", "./lib/*": "./dist/lib/*.js" }, - "files": ["dist/**", "bin/**", "README.md", "package.json"], + "files": [ + "dist/**", + "bin/**", + "README.md", + "package.json" + ], "scripts": { "preinstall": "npx only-allow pnpm", "build": "../node_modules/.bin/tsc --build", "tsc": "../node_modules/.bin/tsc --watch --pretty --preserveWatchOutput", "clean": "rm -rf node_modules dist", - "test": "pnpm exec jest --forceExit --runInBand" + "test": "pnpm exec jest --forceExit --runInBand", + "depcheck": "pnpx depcheck" }, "author": "SageMath, Inc.", - "keywords": ["cocalc", "jupyter"], + "keywords": [ + "cocalc", + "jupyter" + ], "license": "SEE LICENSE.md", "dependencies": { "@cocalc/api-client": "workspace:*", "@cocalc/backend": "workspace:*", "@cocalc/comm": "workspace:*", - "@cocalc/nats": "workspace:*", + "@cocalc/conat": "workspace:*", "@cocalc/sync-client": "workspace:*", "@cocalc/util": "workspace:*", - "execa": "^8.0.1", "lodash": "^4.17.21", "lz4-napi": "^2.8.0", - "mkdirp": "^1.0.4", - "tsimportlib": "^0.0.5" + "mkdirp": "^1.0.4" }, "homepage": "https://github.com/sagemathinc/cocalc/tree/master/src/packages/sync-fs", "repository": { diff --git a/src/packages/sync-fs/tsconfig.json b/src/packages/sync-fs/tsconfig.json index fc9b1d0f49..2290a242a5 100644 --- a/src/packages/sync-fs/tsconfig.json +++ b/src/packages/sync-fs/tsconfig.json @@ -10,7 +10,7 @@ { "path": "../sync-client" }, { "path": "../backend" }, { "path": "../comm" }, - { "path": "../nats" }, + { "path": "../conat" }, { "path": "../util" } ] } diff --git a/src/packages/sync/client/conat-sync-client.ts b/src/packages/sync/client/conat-sync-client.ts new file mode 100644 index 0000000000..9d6a97702f --- /dev/null +++ b/src/packages/sync/client/conat-sync-client.ts @@ -0,0 +1,147 @@ +/* +A SyncClient implementation that ONLY requires a valid Conat Client instance to +work. This makes it possible for any two clients connected to a Conat network to use +a document together collaboratively. + +Any functionality involving the filesystem obviously is a no-op. + +!WORK IN PROGRESS! +*/ + +import { EventEmitter } from "events"; +import { type Client as SyncClient } from "@cocalc/sync/client/types"; +import { SyncTable } from "@cocalc/sync/table/synctable"; +import { once } from "@cocalc/util/async-utils"; +import { FileWatcher } from "@cocalc/sync/editor/string/test/client-test"; +import { type Client as ConatClient } from "@cocalc/conat/core/client"; +import { getLogger } from "@cocalc/conat/client"; +import { parseQueryWithOptions } from "@cocalc/sync/table/util"; +import { PubSub } from "@cocalc/conat/sync/pubsub"; + +const logger = getLogger("conat-sync-client"); + +export class ConatSyncClient extends EventEmitter implements SyncClient { + constructor(private client: ConatClient) { + super(); + } + + synctable_conat = async (query0, options?): Promise => { + const { query } = parseQueryWithOptions(query0, options); + return (await this.client.sync.synctable({ + ...options, + query, + })) as any; + }; + + pubsub_conat = async (opts) => { + return new PubSub({ client: this.client, ...opts }); + }; + + // account_id or project_id + client_id = (): string => { + return this.client.id; + }; + + server_time = (): Date => { + return new Date(); + }; + + isTestClient = () => { + return false; + }; + + is_project = (): boolean => { + return false; + }; + + is_browser = (): boolean => { + // most generic -- no filesystem assumption + return true; + }; + + is_compute_server = (): boolean => { + return false; + }; + + dbg = (f: string): Function => { + return (...args) => logger.debug(f, ...args); + }; + + log_error = (_opts): void => {}; + + query = (_opts): void => {}; + + is_connected = (): boolean => { + return true; + }; + + is_signed_in = (): boolean => { + return true; + }; + + // + // filesystem stuff that is assumed to be defined but not used... + // + mark_file = (_opts: { + project_id: string; + path: string; + action: string; + ttl: number; + }) => {}; + + path_access = (opts: { path: string; mode: string; cb: Function }): void => { + opts.cb(true); + }; + + path_exists = (opts: { path: string; cb: Function }): void => { + opts.cb(true); + }; + path_stat = (opts: { path: string; cb: Function }): void => { + opts.cb(true); + }; + + path_read = async (opts: { + path: string; + maxsize_MB?: number; + cb: Function; + }): Promise => { + opts.cb(true); + }; + write_file = async (opts: { + path: string; + data: string; + cb: Function; + }): Promise => { + opts.cb(true); + }; + watch_file = (opts: { path: string }): FileWatcher => { + return new FileWatcher(opts.path); + }; + + touch_project = (_): void => {}; + + query_cancel = (_): void => {}; + + alert_message = (_): void => {}; + + is_deleted = (_filename: string, _project_id?: string): boolean => { + return false; + }; + + set_deleted = (_filename: string, _project_id?: string): void => {}; + + synctable_ephemeral = async ( + _project_id: string, + query: any, + options: any, + throttle_changes?: number, + ): Promise => { + const s = new SyncTable(query, options, this, throttle_changes); + await once(s, "connected"); + return s; + }; + + sage_session = (_opts): void => {}; + + shell = (_opts): void => {}; +} diff --git a/src/packages/sync/client/sync-client.ts b/src/packages/sync/client/sync-client.ts index f8a4f56952..56db603ed7 100644 --- a/src/packages/sync/client/sync-client.ts +++ b/src/packages/sync/client/sync-client.ts @@ -8,7 +8,7 @@ Functionality related to Sync. */ import { once } from "@cocalc/util/async-utils"; -import { defaults, is_valid_uuid_string, required } from "@cocalc/util/misc"; +import { defaults, required } from "@cocalc/util/misc"; import { SyncDoc, SyncOpts0 } from "@cocalc/sync/editor/generic/sync-doc"; import { SyncDB, SyncDBOpts0 } from "@cocalc/sync/editor/db"; import { SyncString } from "@cocalc/sync/editor/string/sync"; @@ -19,10 +19,8 @@ import { QueryOptions, synctable_no_changefeed, } from "@cocalc/sync/table"; -import synctable_project from "./synctable-project"; -import type { Channel, AppClient } from "./types"; -import { getSyncDocType } from "@cocalc/nats/sync/syncdoc-info"; -// import { refCacheSync } from "@cocalc/util/refcache"; +import type { AppClient } from "./types"; +import { getSyncDocType } from "@cocalc/conat/sync/syncdoc-info"; interface SyncOpts extends Omit { noCache?: boolean; @@ -73,36 +71,6 @@ export class SyncClient { ); } - public async synctable_project( - project_id: string, - query: Query, - options?: QueryOptions, - throttle_changes?: number, - id: string = "", - ): Promise { - return await synctable_project({ - project_id, - query, - options: options ?? [], - client: this.client, - throttle_changes, - id, - }); - } - - // NOT currently used. - public async symmetric_channel( - name: string, - project_id: string, - ): Promise { - if (!is_valid_uuid_string(project_id) || typeof name !== "string") { - throw Error("project_id must be a valid uuid and name must be a string"); - } - return (await this.client.project_client.api(project_id)).symmetric_channel( - name, - ); - } - // These are not working properly, e.g., if you close and open // a LARGE jupyter notebook quickly (so save to disk takes a while), // then it gets broken until browser refresh. The problem is that diff --git a/src/packages/sync/client/synctable-project.ts b/src/packages/sync/client/synctable-project.ts deleted file mode 100644 index c43ee3e85a..0000000000 --- a/src/packages/sync/client/synctable-project.ts +++ /dev/null @@ -1,319 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -/* -Synctable that uses the project websocket rather than the database. -*/ - -import { delay } from "awaiting"; - -import { type SyncTable, synctable_no_database } from "@cocalc/sync/table"; -import { once, retry_until_success } from "@cocalc/util/async-utils"; -import { assertDefined } from "@cocalc/util/misc"; -import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; -import type { AppClient } from "./types"; - -// Always wait at least this long between connect attempts. This -// avoids flooding the project with connection requests if, e.g., the -// client limit for a particular file is reached. -const MIN_CONNECT_WAIT_MS = 5000; - -interface Options { - project_id: string; - query: object; - options: any[]; - client: AppClient; - throttle_changes?: undefined | number; - id: string; -} - -import { EventEmitter } from "events"; - -class SyncTableChannel extends EventEmitter { - public synctable?: SyncTable; - private project_id: string; - private client: AppClient; - private channel?: any; - private websocket?: any; - private query: any; - private options: any; - private key: string; - - private last_connect: number = 0; - - private connected: boolean = false; - - constructor(opts: Options) { - super(); - const { project_id, query, options, client, throttle_changes } = opts; - if (query == null) { - throw Error("query must be defined"); - } - if (options == null) { - throw Error("options must be defined"); - } - this.key = key(opts); - this.synctable = synctable_no_database( - query, - options, - client, - throttle_changes, - [], - project_id, - ); - (this.synctable as any).channel = this; // for debugging - this.project_id = project_id; - this.client = client; - this.query = query; - this.options = options; - this.init_synctable_handlers(); - - this.connect = reuseInFlight(this.connect.bind(this)); - this.log = this.log.bind(this); - this.connect(); - } - - public is_connected(): boolean { - return this.connected; - } - - private log = (..._args) => { - //console.log("SyncTableChannel", this.query, ..._args); - }; - - private async connect(): Promise { - this.log("connect..."); - if (this.synctable == null) return; - this.set_connected(false); - this.clean_up_sockets(); - - const time_since_last_connect = Date.now() - this.last_connect; - if (time_since_last_connect < MIN_CONNECT_WAIT_MS) { - // Last attempt to connect was very recent, so we wait a little before - // trying again. - await delay(MIN_CONNECT_WAIT_MS - time_since_last_connect); - } - - await retry_until_success({ - max_delay: 5000, - f: this.attempt_to_connect.bind(this), - desc: "webapp-synctable-connect", - log: this.log, - }); - - this.last_connect = Date.now(); - } - - private set_connected(connected: boolean): void { - if (this.synctable == null) return; - this.log("set_connected", connected); - this.connected = connected; - if (this.synctable.client.set_connected != null) { - this.synctable.client.set_connected(connected); - } - if (connected) { - this.emit("connected"); - } else { - this.emit("disconnected"); - } - } - // Various things could go wrong, e.g., the websocket breaking - // while trying to get the api synctable_channel, touch - // project might time out, etc. - private async attempt_to_connect(): Promise { - // Start with fresh websocket and channel -- old one may be dead. - this.clean_up_sockets(); - // touch_project mainly makes sure that some hub is connected to - // the project, so the project can do DB queries. Also - // starts the project. - this.client.touch_project(this.project_id); - // Get a websocket. - this.websocket = await this.client.project_client.websocket( - this.project_id, - ); - if (this.websocket.state != "online") { - // give websocket state one chance to change. - // It could change to destroyed or online. - this.log( - "wait for websocket to connect since state is", - this.websocket.state, - ); - await once(this.websocket, "state"); - } - if (this.websocket.state != "online") { - // Already offline... let's try again from the top. - this.log("websocket failed"); - throw Error("websocket went offline already"); - } - - this.log("Get a channel"); - const api = await this.client.project_client.api(this.project_id); - this.channel = await api.synctable_channel(this.query, this.options); - - if (this.websocket.state != "online") { - // Already offline... let's try again from the top. - throw Error("websocket went offline already"); - } - - this.channel.on("data", this.handle_mesg_from_project.bind(this)); - this.websocket.on("offline", this.connect); - this.channel.on("close", this.connect); - } - - private init_synctable_handlers(): void { - assertDefined(this.synctable); - this.synctable.on("timed-changes", (timed_changes) => { - this.send_mesg_to_project({ timed_changes }); - }); - this.synctable.once("closed", this.close.bind(this)); - } - - private clean_up_sockets(): void { - if (this.channel != null) { - this.channel.removeListener("close", this.connect); - - // Explicitly emit end -- this is a hack, - // since this is the only way to force the - // channel clean-up code to run in primus-multiplex, - // and it gets run async later if we don't do this. - // TODO: rewrite primus-multiplex from scratch. - this.channel.emit("end"); - - try { - this.channel.end(); - } catch (err) { - // no op -- this does happen if channel.conn is destroyed - } - delete this.channel; - } - - if (this.websocket != null) { - this.websocket.removeListener("offline", this.connect); - delete this.websocket; - } - } - - private async close(): Promise { - delete cache[this.key]; - this.clean_up_sockets(); - if (this.synctable != null) { - const s = this.synctable; - delete this.synctable; - await s.close(); - } - } - - private handle_mesg_from_project(mesg): void { - this.log("project --> client: ", mesg); - if (this.synctable == null) { - this.log("project --> client: NO SYNCTABLE"); - return; // can happen during close - } - if (mesg == null) { - throw Error("mesg must not be null"); - } - if (mesg.error != null) { - const { alert_message } = this.client; - const message = `Error opening file -- ${ - mesg.error - } -- wait, restart your project or refresh your browser. Query=${JSON.stringify( - this.query, - )}`; - if (alert_message != null) { - alert_message({ type: "info", message, timeout: 10 }); - } else { - console.warn(message); - } - } - if (mesg.event == "message") { - this.synctable.emit("message", mesg.data); - return; - } - if (mesg.init != null) { - this.log("project --> client: init_browser_client"); - this.synctable.init_browser_client(mesg.init); - // after init message, we are now initialized - // and in the connected state. - this.set_connected(true); - } - if (mesg.versioned_changes != null) { - this.log("project --> client: versioned_changes"); - this.synctable.apply_changes_to_browser_client(mesg.versioned_changes); - } - } - - private send_mesg_to_project(mesg): void { - this.log("project <-- client: ", mesg); - if (!this.connected) { - throw Error("must be connected"); - } - if (this.websocket == null) { - throw Error("websocket must not be null"); - } - if (this.channel == null) { - throw Error("channel must not be null"); - } - if (this.websocket.state != "online") { - throw Error( - `websocket state must be online but it is '${this.websocket.state}'`, - ); - } - this.channel.write(mesg); - } -} - -// We use a cache to ensure there is at most one synctable -// at a time with given defining parameters. This is just -// for efficiency and sanity, so we use JSON.stringify instead -// of a guranteed stable json. -const cache: { [key: string]: SyncTableChannel } = {}; - -// ONLY uncomment when developing! -// (window as any).channel_cache = cache; - -// The id here is so that the synctables and channels are unique -// **for a given syncdoc**. There can be multiple syncdocs for -// the same underlying project_id/path, e.g., -// - when timetravel and a document are both open at the same time, -// - when a document is closing (and saving offline changes) at the -// same time that it is being opened; to see this disconnect from -// the network, make changes, clocse the file tab, then open it -// again, and reconnect to the network. -// See https://github.com/sagemathinc/cocalc/issues/3595 for why this -// opts.id below is so important. I tried several different approaches, -// and this is the best by far. -function key(opts: Options): string { - return `${opts.id}-${opts.project_id}-${JSON.stringify( - opts.query, - )}-${JSON.stringify(opts.options)}`; -} - -// NOTE: This function can be called by a LOT of different things at once whenever -// waiting to connect to a project. The "await once" inside it creates -// a listener on SyncTableChannel, and there is a limit on the number of -// those you can create without raising a limit (that was appearing in the -// console log a lot). Thus our use of reuseInFlight to prevent this. -async function synctable_project0(opts: Options): Promise { - const k = key(opts); - // console.log("key = ", k); - let t; - if (cache[k] !== undefined) { - t = cache[k]; - } else { - t = new SyncTableChannel(opts); - cache[k] = t; - } - if (!t.is_connected()) { - await once(t, "connected"); - } - return t.synctable; -} - -const synctable_project = reuseInFlight(synctable_project0, { - createKey: (args) => - JSON.stringify([args[0].project_id, args[0].query, args[0].options]), -}); - -export default synctable_project; diff --git a/src/packages/sync/client/types.ts b/src/packages/sync/client/types.ts index dcea22e042..19015f8107 100644 --- a/src/packages/sync/client/types.ts +++ b/src/packages/sync/client/types.ts @@ -1,9 +1,9 @@ import type { EventEmitter } from "events"; import type { CB } from "@cocalc/util/types/callback"; import type { - CallNatsServiceFunction, - CreateNatsServiceFunction, -} from "@cocalc/nats/service"; + CallConatServiceFunction, + CreateConatServiceFunction, +} from "@cocalc/conat/service"; // What we need the client to implement so we can use // it to support a table. @@ -21,8 +21,8 @@ export interface Client extends EventEmitter { touch_project: (project_id: string, compute_server_id?: number) => void; set_connected?: Function; is_deleted: (path: string, project_id: string) => true | false | undefined; - callNatsService?: CallNatsServiceFunction; - createNatsService?: CreateNatsServiceFunction; + callConatService?: CallConatServiceFunction; + createConatService?: CreateConatServiceFunction; client_id?: () => string | undefined; } @@ -92,11 +92,6 @@ export interface ProjectWebsocket extends EventEmitter { } export interface API { - symmetric_channel(name: string): Promise; - synctable_channel( - query: { [field: string]: any }, - options: { [field: string]: any }[], - ): Promise; version(): Promise; } diff --git a/src/packages/sync/editor/generic/evaluator.ts b/src/packages/sync/editor/generic/evaluator.ts index be39e9774b..c538ad707f 100644 --- a/src/packages/sync/editor/generic/evaluator.ts +++ b/src/packages/sync/editor/generic/evaluator.ts @@ -57,10 +57,7 @@ export class Evaluator { this.syncdoc = syncdoc; this.client = client; this.create_synctable = create_synctable; - if (this.syncdoc.data_server == "project") { - // options only supported for project... - this.table_options = [{ ephemeral: true, persistent: true }]; - } + this.table_options = [{ ephemeral: true, persistent: true }]; } public async init(): Promise { diff --git a/src/packages/sync/editor/generic/ipywidgets-state.ts b/src/packages/sync/editor/generic/ipywidgets-state.ts index 61eadabf58..3a75a41802 100644 --- a/src/packages/sync/editor/generic/ipywidgets-state.ts +++ b/src/packages/sync/editor/generic/ipywidgets-state.ts @@ -102,12 +102,7 @@ export class IpywidgetsState extends EventEmitter { this.syncdoc = syncdoc; this.client = client; this.create_synctable = create_synctable; - if (this.syncdoc.data_server == "project") { - // options only supported for project... - // ephemeral -- don't store longterm in database - // persistent -- doesn't automatically vanish when all browser clients disconnect - this.table_options = [{ ephemeral: true, persistent: true }]; - } + this.table_options = [{ ephemeral: true }]; this.gc = !DISABLE_GC && client.is_project() // no-op if not project or DISABLE_GC ? debounce(() => { diff --git a/src/packages/sync/editor/generic/legacy.ts b/src/packages/sync/editor/generic/legacy.ts index 70fa0647d6..9165f21f98 100644 --- a/src/packages/sync/editor/generic/legacy.ts +++ b/src/packages/sync/editor/generic/legacy.ts @@ -3,7 +3,7 @@ Support legacy TimeTravel history from before the switch to NATS. */ import { type Client } from "./types"; -import { type DB } from "@cocalc/nats/hub-api/db"; +import { type DB } from "@cocalc/conat/hub/api/db"; export interface LegacyPatch { time: Date; @@ -30,7 +30,7 @@ export class LegacyHistory { path: string; }) { // this is only available on the frontend browser, which is all that matters. - this.db = (client as any).nats_client?.hub.db as any; + this.db = (client as any).conat_client?.hub.db as any; this.project_id = project_id; this.path = path; } @@ -57,8 +57,8 @@ export class LegacyHistory { return { patches: [], users: [] }; } const s = await this.db.getLegacyTimeTravelPatches({ - requestMany: true, // since response may be large - timeout: 60000, + // long timeout, since response may be large or take a while to pull out of cold storage + timeout: 90000, uuid: info.uuid, }); let patches; diff --git a/src/packages/sync/editor/generic/sorted-patch-list.ts b/src/packages/sync/editor/generic/sorted-patch-list.ts index 82a2d6aad5..3d794c40dc 100644 --- a/src/packages/sync/editor/generic/sorted-patch-list.ts +++ b/src/packages/sync/editor/generic/sorted-patch-list.ts @@ -253,6 +253,7 @@ export class SortedPatchList extends EventEmitter { delete this.versions_cache; this.patches = this.patches.concat(newPatches); this.patches.sort(patch_cmp); + this.emit("change"); } else { // nothing moved from staging to live, so **converged**. return; diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index 5a1dc48e44..78e1bb24c0 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -19,7 +19,7 @@ EVENTS: - ... TODO */ -const USE_NATS = true; +const USE_CONAT = true; /* OFFLINE_THRESH_S - If the client becomes disconnected from the backend for more than this long then---on reconnect---do @@ -82,6 +82,7 @@ import { once, retry_until_success, reuse_in_flight_methods, + until, } from "@cocalc/util/async-utils"; import { wait } from "@cocalc/util/async-wait"; import { @@ -94,7 +95,6 @@ import { hash_string, keys, minutes_ago, - uuid, } from "@cocalc/util/misc"; import * as schema from "@cocalc/util/schema"; import { delay } from "awaiting"; @@ -114,12 +114,13 @@ import type { Patch, } from "./types"; import { isTestClient, patch_cmp } from "./util"; -import { NATS_OPEN_FILE_TOUCH_INTERVAL } from "@cocalc/util/nats"; +import { CONAT_OPEN_FILE_TOUCH_INTERVAL } from "@cocalc/util/conat"; import mergeDeep from "@cocalc/util/immutable-deep-merge"; import { JUPYTER_SYNCDB_EXTENSIONS } from "@cocalc/util/jupyter/names"; import { LegacyHistory } from "./legacy"; -import { waitUntilConnected } from "@cocalc/nats/util"; -import { getLogger } from "@cocalc/nats/client"; +import { getLogger } from "@cocalc/conat/client"; + +const DEBUG = false; export type State = "init" | "ready" | "closed"; export type DataServer = "project" | "database"; @@ -165,9 +166,7 @@ export interface UndoState { } // NOTE: Do not make multiple SyncDoc's for the same document, especially -// not on the frontend. Proper reference counted handling of this is done -// at sync/client/sync-client.ts, or in some applications where you can easily -// be sure there is only reference, just be sure. +// not on the frontend. const logger = getLogger("sync-doc"); logger.debug("init"); @@ -178,9 +177,6 @@ export class SyncDoc extends EventEmitter { private string_id: string; private my_user_id: number; - // This id is used for equality test and caching. - private id: string = uuid(); - private client: Client; private _from_str: (str: string) => Document; // creates a doc from a string. @@ -254,7 +250,6 @@ export class SyncDoc extends EventEmitter { private save_to_disk_end_ctime: number | undefined; private persistent: boolean = false; - public readonly data_server: DataServer = "project"; private last_has_unsaved_changes?: boolean = undefined; @@ -266,7 +261,7 @@ export class SyncDoc extends EventEmitter { // static because we want exactly one across all docs! private static computeServerManagerDoc?: SyncDoc; - private useNats: boolean; + private useConat: boolean; legacy: LegacyHistory; constructor(opts: SyncOpts) { @@ -302,10 +297,10 @@ export class SyncDoc extends EventEmitter { client: this.client, }); - // NOTE: Do not use nats in test mode, since there we use a minimal + // NOTE: Do not use conat in test mode, since there we use a minimal // "fake" client that does all communication internally and doesn't - // use nats. We also use this for the messages composer. - this.useNats = USE_NATS && !isTestClient(opts.client); + // use conat. We also use this for the messages composer. + this.useConat = USE_CONAT && !isTestClient(opts.client); if (this.ephemeral) { // So the doctype written to the database reflects the // ephemeral state. Here ephemeral determines whether @@ -352,44 +347,35 @@ export class SyncDoc extends EventEmitter { until it is (however long, etc.). If this fails, it closes this SyncDoc. */ - private init = async (): Promise => { + private initialized = false; + private init = async () => { + if (this.initialized) { + throw Error("init can only be called once"); + } + // const start = Date.now(); this.assert_not_closed("init"); const log = this.dbg("init"); - - let d = 3000; - while (this.state == "init") { - try { - //const t0 = new Date(); - - log("initializing all tables..."); - if (this.useNats) { - await waitUntilConnected(); + await until( + async () => { + if (this.state != "init") { + return true; } - await this.initAll(); - log("initAll succeeded"); - // got it! - break; - - //console.log( - // `time to open file ${this.path}: ${Date.now() - t0.valueOf()}` - //); - } catch (err) { - const m = `WARNING: problem initializing ${this.path} -- ${err}`; - log(m); - // log always: - console.log(m); - // @ts-ignore - if (this.state == "closed") { - log("init", this.path, "state closed so exit"); - // completely normal that this could happen on frontend - it just means - // that we closed the file before finished opening it... - return; + try { + log("initializing all tables..."); + await this.initAll(); + log("initAll succeeded"); + return true; + } catch (err) { + const m = `WARNING: problem initializing ${this.path} -- ${err}`; + log(m); + // log always: + console.log(m); } - log(`wait ${d} then try again`); - await delay(d); - d = Math.min(d * 1.3, 15000); - } - } + log("wait then try again"); + return false; + }, + { start: 3000, max: 15000, decay: 1.3 }, + ); // Success -- everything initialized with no issues. this.set_state("ready"); @@ -409,6 +395,7 @@ export class SyncDoc extends EventEmitter { // on the result, you need to clear it when the state // changes. See the function handleComputeServerManagerChange. private isFileServer = reuseInFlight(async () => { + if (this.state == "closed") return; if (this.client.is_browser()) { // browser is never the file server (yet), and doesn't need to do // anything related to watching for changes in state. @@ -617,7 +604,7 @@ export class SyncDoc extends EventEmitter { // table not initialized yet return; } - if (this.useNats) { + if (this.useConat) { const time = this.client.server_time().valueOf(); const x: { user_id: number; @@ -660,7 +647,7 @@ export class SyncDoc extends EventEmitter { set_cursor_locs: typeof this.setCursorLocsNoThrottle = throttle( this.setCursorLocsNoThrottle, - USE_NATS ? CURSOR_THROTTLE_NATS_MS : CURSOR_THROTTLE_MS, + USE_CONAT ? CURSOR_THROTTLE_NATS_MS : CURSOR_THROTTLE_MS, { leading: true, trailing: true, @@ -1032,11 +1019,14 @@ export class SyncDoc extends EventEmitter { this.assert_table_is_ready("syncstring"); this.dbg("set_initialized")({ error, read_only, size }); const init = { time: this.client.server_time(), size, error }; - await this.set_syncstring_table({ - init, - read_only, - last_active: this.client.server_time(), - }); + for (let i = 0; i < 3; i++) { + await this.set_syncstring_table({ + init, + read_only, + last_active: this.client.server_time(), + }); + await delay(1000); + } }; /* List of logical timestamps of the versions of this string in the sync @@ -1093,14 +1083,9 @@ export class SyncDoc extends EventEmitter { } } - // Close synchronized editing of this string; this stops listening - // for changes and stops broadcasting changes. - close = reuseInFlight(async () => { - if (this.state == "closed") { - return; - } - const dbg = this.dbg("close"); - dbg("close"); + // more gentle version -- this can cause the project actions + // to be *created* etc. + end = reuseInFlight(async () => { if (this.client.is_browser() && this.state == "ready") { try { await this.save_to_disk(); @@ -1110,6 +1095,18 @@ export class SyncDoc extends EventEmitter { // Do nothing here. } } + this.close(); + }); + + // Close synchronized editing of this string; this stops listening + // for changes and stops broadcasting changes. + close = reuseInFlight(async () => { + if (this.state == "closed") { + return; + } + const dbg = this.dbg("close"); + dbg("close"); + SyncDoc.computeServerManagerDoc?.removeListener( "change", this.handleComputeServerManagerChange, @@ -1166,16 +1163,11 @@ export class SyncDoc extends EventEmitter { this.patch_list.close(); } - // - // ASYNC STUFF - in particular, these may all - // attempt to do some last attempt to send changes - // to the database. - // try { - await this.async_close(); - dbg("async_close -- successfully saved all data to database"); + this.closeTables(); + dbg("closeTables -- successfully saved all data to database"); } catch (err) { - dbg(`async_close -- ERROR -- ${err}`); + dbg(`closeTables -- ERROR -- ${err}`); } // this avoids memory leaks: close(this); @@ -1185,36 +1177,12 @@ export class SyncDoc extends EventEmitter { dbg("close done"); }); - private async_close = async () => { - const promises: Promise[] = []; - - if (this.syncstring_table != null) { - promises.push(this.syncstring_table.close()); - } - - if (this.patches_table != null) { - promises.push(this.patches_table.close()); - } - - if (this.cursors_table != null) { - promises.push(this.cursors_table.close()); - } - - if (this.evaluator != null) { - promises.push(this.evaluator.close()); - } - - if (this.ipywidgets_state != null) { - promises.push(this.ipywidgets_state.close()); - } - - const results = await Promise.allSettled(promises); - - results.forEach((result) => { - if (result.status === "rejected") { - throw Error(result.reason); - } - }); + private closeTables = async () => { + this.syncstring_table?.close(); + this.patches_table?.close(); + this.cursors_table?.close(); + this.evaluator?.close(); + this.ipywidgets_state?.close(); }; // TODO: We **have** to do this on the client, since the backend @@ -1238,7 +1206,7 @@ export class SyncDoc extends EventEmitter { // patches table uses the string_id, which is a SHA1 hash. private ensure_syncstring_exists_in_db = async (): Promise => { const dbg = this.dbg("ensure_syncstring_exists_in_db"); - if (this.useNats) { + if (this.useConat) { dbg("skipping -- no database"); return; } @@ -1277,22 +1245,24 @@ export class SyncDoc extends EventEmitter { ): Promise => { this.assert_not_closed("synctable"); const dbg = this.dbg("synctable"); - if ( - !this.useNats && - !this.ephemeral && - this.persistent && - this.data_server == "project" - ) { + if (!this.useConat && !this.ephemeral && this.persistent) { // persistent table in a non-ephemeral syncdoc, so ensure that table is // persisted to database (not just in memory). options = options.concat([{ persistent: true }]); } - if (this.ephemeral && this.data_server == "project") { + if (this.ephemeral) { options.push({ ephemeral: true }); } let synctable; - if (this.useNats && query.patches) { - synctable = await this.client.synctable_nats(query, { + let ephemeral = false; + for (const x of options) { + if (x.ephemeral) { + ephemeral = true; + break; + } + } + if (this.useConat && query.patches) { + synctable = await this.client.synctable_conat(query, { obj: { project_id: this.project_id, path: this.path, @@ -1301,9 +1271,10 @@ export class SyncDoc extends EventEmitter { atomic: true, desc: { path: this.path }, start_seq: this.last_seq, + ephemeral, }); - } else if (this.useNats && query.syncstrings) { - synctable = await this.client.synctable_nats(query, { + } else if (this.useConat && query.syncstrings) { + synctable = await this.client.synctable_conat(query, { obj: { project_id: this.project_id, path: this.path, @@ -1312,9 +1283,10 @@ export class SyncDoc extends EventEmitter { atomic: false, immutable: true, desc: { path: this.path }, + ephemeral, }); - } else if (this.useNats && query.ipywidgets) { - synctable = await this.client.synctable_nats(query, { + } else if (this.useConat && query.ipywidgets) { + synctable = await this.client.synctable_conat(query, { obj: { project_id: this.project_id, path: this.path, @@ -1323,13 +1295,13 @@ export class SyncDoc extends EventEmitter { atomic: true, immutable: true, // for now just putting a 1-day limit on the ipywidgets table - // so we don't waste a ton of space. TODO: We could to also clear this - // table on halt, startup, etc. - limits: { max_age: 1000 * 60 * 60 * 24 }, + // so we don't waste a ton of space. + config: { max_age: 1000 * 60 * 60 * 24 }, desc: { path: this.path }, + ephemeral: true, // ipywidgets state always ephemeral }); - } else if (this.useNats && (query.eval_inputs || query.eval_outputs)) { - synctable = await this.client.synctable_nats(query, { + } else if (this.useConat && (query.eval_inputs || query.eval_outputs)) { + synctable = await this.client.synctable_conat(query, { obj: { project_id: this.project_id, path: this.path, @@ -1337,11 +1309,12 @@ export class SyncDoc extends EventEmitter { stream: false, atomic: true, immutable: true, - limits: { max_age: 30000 }, + config: { max_age: 5 * 60 * 1000 }, desc: { path: this.path }, + ephemeral: true, // eval state (for sagews) is always ephemeral }); - } else if (this.useNats) { - synctable = await this.client.synctable_nats(query, { + } else if (this.useConat) { + synctable = await this.client.synctable_conat(query, { obj: { project_id: this.project_id, path: this.path, @@ -1350,31 +1323,19 @@ export class SyncDoc extends EventEmitter { atomic: true, immutable: true, desc: { path: this.path }, + ephemeral, }); } else { - switch (this.data_server) { - case "project": - synctable = await this.client.synctable_project( - this.project_id, - query, - options, - throttle_changes, - this.id, - ); - break; - case "database": - if (this.client.synctable_database == null) { - throw Error("database server not supported by project"); - } - synctable = await this.client.synctable_database?.( - query, - options, - throttle_changes, - ); - break; - default: - throw Error(`uknown server ${this.data_server}`); - } + // only used for unit tests and the ephemeral messaging composer + if (this.client.synctable_ephemeral == null) { + throw Error(`client does not support sync properly`); + } + synctable = await this.client.synctable_ephemeral( + this.project_id, + query, + options, + throttle_changes, + ); } // We listen and log error events. This is useful because in some settings, e.g., // in the project, an eventemitter with no listener for errors, which has an error, @@ -1414,37 +1375,21 @@ export class SyncDoc extends EventEmitter { doctype: JSON.stringify(this.doctype), }); } else { - dbg("waiting for, then handling the first update..."); - await this.handle_syncstring_update(); + dbg("handling the first update..."); + this.handle_syncstring_update(); } - this.syncstring_table.on( - "change", - this.handle_syncstring_update.bind(this), - ); - - // Wait until syncstring is not archived -- if we open an - // older syncstring, the patches may be archived, - // and we have to wait until - // after they have been pulled from blob storage before - // we init the patch table, load from disk, etc. - const is_not_archived: () => boolean = () => { - const ss = this.syncstring_table_get_one(); - if (ss != null) { - return !ss.get("archived"); - } else { - return false; - } - }; - dbg("waiting for syncstring to be not archived"); - await this.syncstring_table.wait(is_not_archived, 120); + this.syncstring_table.on("change", this.handle_syncstring_update); }; // Used for internal debug logging private dbg = (_f: string = ""): Function => { - return (..._args) => {}; - // return (...args) => { - // logger.debug(this.path, _f, ...args); - // }; + if (DEBUG) { + return (...args) => { + logger.debug(this.path, _f, ...args); + }; + } else { + return (..._args) => {}; + } }; private initAll = async (): Promise => { @@ -1460,9 +1405,8 @@ export class SyncDoc extends EventEmitter { this.assert_not_closed("initAll -- before ensuring syncstring exists"); await this.ensure_syncstring_exists_in_db(); - log("syncstring_table"); - this.assert_not_closed("initAll -- before init_syncstring_table"); - await this.init_syncstring_table(); + await this.init_syncstring_table(), + this.assert_not_closed("initAll -- successful init_syncstring_table"); log("patch_list, cursors, evaluator, ipywidgets"); this.assert_not_closed( @@ -1513,11 +1457,16 @@ export class SyncDoc extends EventEmitter { desc: "syncdoc -- load_from_disk", }); log("done loading from disk"); - this.assert_not_closed("initAll -- load from disk"); + } else { + if (this.patch_list!.count() == 0) { + await Promise.race([ + this.waitUntilFullyReady(), + once(this.patch_list!, "change"), + ]); + } } - - log("wait_until_fully_ready"); - await this.wait_until_fully_ready(); + this.assert_not_closed("initAll -- load from disk"); + this.emit("init"); this.assert_not_closed("initAll -- after waiting until fully ready"); @@ -1543,45 +1492,33 @@ export class SyncDoc extends EventEmitter { // wait until the syncstring table is ready to be // used (so extracted from archive, etc.), - private wait_until_fully_ready = async (): Promise => { + private waitUntilFullyReady = async (): Promise => { this.assert_not_closed("wait_until_fully_ready"); const dbg = this.dbg("wait_until_fully_ready"); dbg(); if (this.client.is_browser() && this.init_error()) { - // init is set and is in error state. Give the backend 3 seconds + // init is set and is in error state. Give the backend a few seconds // to try to fix this error before giving up. The browser client // can close and open the file to retry this (as instructed). try { - await this.syncstring_table.wait(() => !this.init_error(), 3); + await this.syncstring_table.wait(() => !this.init_error(), 5); } catch (err) { // fine -- let the code below deal with this problem... } } - const is_init_and_not_archived = (t: SyncTable) => { - this.assert_not_closed("is_init_and_not_archived"); + const is_init = (t: SyncTable) => { + this.assert_not_closed("is_init"); const tbl = t.get_one(); if (tbl == null) { dbg("null"); return false; } - // init must be set in table and archived must NOT be - // set, so patches are loaded from blob store. - const init = tbl.get("init"); - if (init && !tbl.get("archived")) { - dbg("good to go"); - return init.toJS(); - } else { - dbg("not init yet"); - return false; - } + return tbl.get("init") != null; }; dbg("waiting for init..."); - const init = await this.syncstring_table.wait( - is_init_and_not_archived.bind(this), - 0, - ); + const init = await this.syncstring_table.wait(is_init, 0); dbg("init done"); if (init.error) { throw Error(init.error); @@ -1610,17 +1547,16 @@ export class SyncDoc extends EventEmitter { dbg("handled update queue"); } } - this.emit("init"); }; - private assert_table_is_ready(table: string): void { + private assert_table_is_ready = (table: string): void => { const t = this[table + "_table"]; // not using string template only because it breaks codemirror! if (t == null || t.get_state() != "connected") { throw Error( `Table ${table} must be connected. string_id=${this.string_id}`, ); } - } + }; assert_is_ready = (desc: string): void => { if (this.state != "ready") { @@ -1793,7 +1729,7 @@ export class SyncDoc extends EventEmitter { private patch_table_query = (cutoff?: number) => { const query = { string_id: this.string_id, - is_snapshot: false, // only used with nats + is_snapshot: false, // only used with conat time: cutoff ? { ">=": cutoff } : null, wall: null, // compressed format patch as a JSON *string* @@ -1843,7 +1779,7 @@ export class SyncDoc extends EventEmitter { this.assert_not_closed("init_patch_list -- after making synctable"); const update_has_unsaved_changes = debounce( - this.update_has_unsaved_changes.bind(this), + this.update_has_unsaved_changes, 500, { leading: true, trailing: true }, ); @@ -1947,9 +1883,9 @@ export class SyncDoc extends EventEmitter { dbg("done -- do not care about cursors for this syncdoc."); return; } - if (this.useNats) { - dbg("NATS cursors support using pub/sub"); - this.cursors_table = await this.client.pubsub_nats({ + if (this.useConat) { + dbg("cursors broadcast using pub/sub"); + this.cursors_table = await this.client.pubsub_conat({ project_id: this.project_id, path: this.path, name: "cursors", @@ -1993,12 +1929,7 @@ export class SyncDoc extends EventEmitter { // need to persist it to the database, obviously! // Also, queue_size:1 makes it so only the last cursor position is // saved, e.g., in case of disconnect and reconnect. - let options; - if (this.data_server == "project") { - options = [{ ephemeral: true }, { queue_size: 1 }]; - } else { - options = []; - } + const options = [{ ephemeral: true }, { queue_size: 1 }]; // probably deprecated this.cursors_table = await this.synctable(query, options, 1000); this.assert_not_closed("init_cursors -- after making synctable"); @@ -2197,6 +2128,9 @@ export class SyncDoc extends EventEmitter { return; } await this.handle_patch_update_queue(); + if (this.state != "ready") { + return; + } // Ensure all patches are saved to backend. // We do this after the above, so that creating the newest patch @@ -2265,13 +2199,13 @@ export class SyncDoc extends EventEmitter { // active, so we check if we should make a snapshot. There is the // potential of a race condition where more than one clients make // a snapshot at the same time -- this would waste a little space - // in the nats jetstream, but is otherwise harmless, since the snapshots + // in the stream, but is otherwise harmless, since the snapshots // are identical. this.snapshot_if_necessary(); }; private dstream = () => { - // @ts-ignore -- in general patches_table might not be a nats one still, + // @ts-ignore -- in general patches_table might not be a conat one still, // or at least dstream is an internal implementation detail. const { dstream } = this.patches_table ?? {}; if (dstream == null) { @@ -2280,11 +2214,11 @@ export class SyncDoc extends EventEmitter { return dstream; }; - // return the NATS sequence number of the oldest entry in the + // return the conat-assigned sequence number of the oldest entry in the // patch list with the given time, and also: // - prev_seq -- the sequence number of previous patch before that, for use in "load more" // - index -- the global index of the entry with the given time. - private natsSnapshotSeqInfo = ( + private conatSnapshotSeqInfo = ( time: number, ): { seq: number; prev_seq?: number } => { const dstream = this.dstream(); @@ -2339,7 +2273,7 @@ export class SyncDoc extends EventEmitter { const snapshot: string = this.patch_list.value({ time }).to_str(); // save the snapshot itself in the patches table. - const seq_info = this.natsSnapshotSeqInfo(time); + const seq_info = this.conatSnapshotSeqInfo(time); const obj = { size: snapshot.length, string_id: this.string_id, @@ -2727,7 +2661,7 @@ export class SyncDoc extends EventEmitter { // Brand new syncstring // TODO: worry about race condition with everybody making themselves - // have user_id 0... ? + // have user_id 0... and also setting doctype. this.my_user_id = 0; this.users = [this.client.client_id()]; const obj = { @@ -2754,11 +2688,7 @@ export class SyncDoc extends EventEmitter { if (this.path == null) { // We just opened the file -- emit a load time estimate. - if (x.archived) { - this.emit("load-time-estimate", { type: "archived", time: 3 }); - } else { - this.emit("load-time-estimate", { type: "ready", time: 1 }); - } + this.emit("load-time-estimate", { type: "ready", time: 1 }); } // TODO: handle doctype change here (?) this.setLastSnapshot(x.last_snapshot); @@ -3092,6 +3022,7 @@ export class SyncDoc extends EventEmitter { const time = this.next_patch_time(); this.commit_patch(time, patch); this.save(); // so eventually also gets sent out. + this.touchProject(); return true; }; @@ -3148,12 +3079,14 @@ export class SyncDoc extends EventEmitter { try { await this.save_to_disk_aux(); } catch (err) { + if (this.state != "ready") return; const error = `save to disk failed -- ${err}`; dbg(error); if (await this.isFileServer()) { this.set_save({ error, state: "done" }); } } + if (this.state != "ready") return; if (!(await this.isFileServer())) { dbg("now wait for the save to disk to finish"); @@ -3565,17 +3498,41 @@ export class SyncDoc extends EventEmitter { } }; + // this keeps the project from idle timing out -- it happens + // whenever there is an edit to the file by a browser, and + // keeps the project from stopping. + private touchProject = throttle(() => { + if (this.client?.is_browser()) { + this.client.touch_project?.(this.path); + } + }, 60000); + private initInterestLoop = async () => { - if (!this.client.is_browser() || this.client.touchOpenFile == null) { + if (!this.client.is_browser()) { // only browser clients -- so actual humans return; } - while (this.state != "closed") { + const touch = async () => { + if (this.state == "closed" || this.client?.touchOpenFile == null) return; await this.client.touchOpenFile({ path: this.path, project_id: this.project_id, + doctype: this.doctype, }); - await delay(NATS_OPEN_FILE_TOUCH_INTERVAL); - } + }; + // then every CONAT_OPEN_FILE_TOUCH_INTERVAL (30 seconds). + await until( + async () => { + if (this.state == "closed") { + return true; + } + await touch(); + return false; + }, + { + start: CONAT_OPEN_FILE_TOUCH_INTERVAL, + max: CONAT_OPEN_FILE_TOUCH_INTERVAL, + }, + ); }; } diff --git a/src/packages/sync/editor/generic/types.ts b/src/packages/sync/editor/generic/types.ts index c7a5afd1dc..4cf3519e94 100644 --- a/src/packages/sync/editor/generic/types.ts +++ b/src/packages/sync/editor/generic/types.ts @@ -13,9 +13,9 @@ import { SyncTable } from "@cocalc/sync/table/synctable"; import type { ExecuteCodeOptionsWithCallback } from "@cocalc/util/types/execute-code"; import type { - CallNatsServiceFunction, - CreateNatsServiceFunction, -} from "@cocalc/nats/service"; + CallConatServiceFunction, + CreateConatServiceFunction, +} from "@cocalc/conat/service"; export interface Patch { // time = LOGICAL time of when patch made; this used to be ms since the epoch, but just @@ -120,7 +120,7 @@ export interface ProjectClient extends EventEmitter { watch_file: (opts: { path: string }) => FileWatcher; - synctable_project: ( + synctable_ephemeral?: ( project_id: string, query: any, options: any, @@ -128,10 +128,10 @@ export interface ProjectClient extends EventEmitter { id?: string, ) => Promise; - synctable_nats: (query: any, obj?) => Promise; - pubsub_nats: (query: any, obj?) => Promise; - callNatsService?: CallNatsServiceFunction; - createNatsService?: CreateNatsServiceFunction; + synctable_conat: (query: any, obj?) => Promise; + pubsub_conat: (query: any, obj?) => Promise; + callConatService?: CallConatServiceFunction; + createConatService?: CreateConatServiceFunction; // account_id or project_id or compute_server_id (encoded as a UUID - use decodeUUIDtoNum to decode) client_id: () => string; @@ -165,17 +165,17 @@ export interface Client extends ProjectClient { ttl: number; }) => void; - synctable_database?: ( - query: any, - options: any, - throttle_changes?: number, - ) => Promise; - shell: (opts: ExecuteCodeOptionsWithCallback) => void; sage_session: (opts: { path: string }) => any; - touchOpenFile?: (opts: { project_id: string; path: string }) => Promise; + touchOpenFile?: (opts: { + project_id: string; + path: string; + doctype?; + }) => Promise; + + touch_project?: (path: string) => void; } export interface DocType { diff --git a/src/packages/sync/editor/string/test/client-test.ts b/src/packages/sync/editor/string/test/client-test.ts index ec11ed862f..b9d43e012a 100644 --- a/src/packages/sync/editor/string/test/client-test.ts +++ b/src/packages/sync/editor/string/test/client-test.ts @@ -9,13 +9,13 @@ Minimal client class that we use for testing. import { EventEmitter } from "events"; import { bind_methods, keys } from "@cocalc/util/misc"; -import { once } from "@cocalc/util/async-utils"; import { Client as Client0, FileWatcher as FileWatcher0, } from "../../generic/types"; import { SyncTable } from "@cocalc/sync/table/synctable"; import { ExecuteCodeOptionsWithCallback } from "@cocalc/util/types/execute-code"; +import { once } from "@cocalc/util/async-utils"; export class FileWatcher extends EventEmitter implements FileWatcher0 { private path: string; @@ -156,7 +156,7 @@ export class Client extends EventEmitter implements Client0 { public set_deleted(_filename: string, _project_id?: string): void {} - async synctable_project( + async synctable_ephemeral( _project_id: string, query: any, options: any, @@ -167,23 +167,11 @@ export class Client extends EventEmitter implements Client0 { return s; } - async synctable_database( - _query: any, - _options: any, - _throttle_changes?: number, - ): Promise { - throw Error("synctable_database: not implemented"); + async synctable_conat(_query: any): Promise { + throw Error("synctable_conat: not implemented"); } - - async synctable_nats(_query: any): Promise { - throw Error("synctable_nats: not implemented"); - } - async pubsub_nats(_query: any): Promise { - throw Error("pubsub_nats: not implemented"); - } - - async natsRequest(_subject: string, _mesg: any, _options?) { - throw Error("natsRequest: not implemented"); + async pubsub_conat(_query: any): Promise { + throw Error("pubsub_conat: not implemented"); } // account_id or project_id diff --git a/src/packages/sync/package.json b/src/packages/sync/package.json index c9b1d056b7..5116431d40 100644 --- a/src/packages/sync/package.json +++ b/src/packages/sync/package.json @@ -13,6 +13,7 @@ "build": "../node_modules/.bin/tsc --build", "tsc": "../node_modules/.bin/tsc --watch --pretty --preserveWatchOutput", "test": "pnpm exec jest", + "depcheck": "pnpx depcheck --ignores events", "prepublishOnly": "pnpm test" }, "files": ["dist/**", "bin/**", "README.md", "package.json"], @@ -20,12 +21,11 @@ "keywords": ["cocalc", "realtime synchronization"], "license": "SEE LICENSE.md", "dependencies": { - "@cocalc/nats": "workspace:*", + "@cocalc/conat": "workspace:*", "@cocalc/sync": "workspace:*", "@cocalc/util": "workspace:*", "async": "^1.5.2", "awaiting": "^3.0.0", - "debug": "^4.4.0", "events": "3.3.0", "immutable": "^4.3.0", "json-stable-stringify": "^1.0.1", @@ -38,7 +38,6 @@ "url": "https://github.com/sagemathinc/cocalc" }, "devDependencies": { - "@types/debug": "^4.1.12", "@types/lodash": "^4.14.202", "@types/node": "^18.16.14", "ts-jest": "^29.2.3" diff --git a/src/packages/sync/table/changefeed-conat.ts b/src/packages/sync/table/changefeed-conat.ts new file mode 100644 index 0000000000..d462ae3dab --- /dev/null +++ b/src/packages/sync/table/changefeed-conat.ts @@ -0,0 +1,99 @@ +/* + * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. + * License: MS-RSL – see LICENSE.md for details + */ + +import { EventEmitter } from "events"; +import { changefeed, type Changefeed } from "@cocalc/conat/hub/changefeeds"; +import { conat } from "@cocalc/conat/client"; + +// low level debugging of changefeeds +const LOW_LEVEL_DEBUG = false; +const log = LOW_LEVEL_DEBUG + ? (...args) => { + console.log("changefeed: ", ...args); + } + : (..._args) => {}; + +export class ConatChangefeed extends EventEmitter { + private account_id: string; + private query; + private options; + private state: "disconnected" | "connected" | "closed" = "disconnected"; + private cf?: Changefeed; + + constructor({ + account_id, + query, + options, + }: { + account_id: string; + query; + options?; + }) { + super(); + this.account_id = account_id; + this.query = query; + this.options = options; + } + + log = (...args) => { + if (!LOW_LEVEL_DEBUG) return; + log(this.query, ...args); + }; + + connect = async () => { + this.log("connecting..."); + this.cf = changefeed({ + client: await conat(), + account_id: this.account_id, + query: this.query, + options: this.options, + }); + const { value, done } = await this.cf.next(); + if (done) { + this.log("closed before receiving any values"); + this.close(); + return; + } + this.log("connected"); + this.state = "connected"; + this.watch(); + return value[Object.keys(value)[0]]; + }; + + close = (): void => { + this.log("close"); + if (this.state == "closed") { + return; + } + this.cf?.close(); + delete this.cf; + this.state = "closed"; + this.emit("close"); // yes "close" not "closed" ;-( + }; + + get_state = (): string => { + return this.state; + }; + + private watch = async () => { + if (this.cf == null || this.state == "closed") { + return; + } + try { + for await (const x of this.cf) { + // this.log("got message ", x); + // @ts-ignore + if (this.state == "closed") { + return; + } + this.emit("update", x); + } + } catch (err) { + this.log("got error", err); + } + this.log("watch ended", this.query); + this.close(); + }; +} diff --git a/src/packages/sync/table/changefeed-nats.ts b/src/packages/sync/table/changefeed-nats.ts deleted file mode 100644 index 5f93084a46..0000000000 --- a/src/packages/sync/table/changefeed-nats.ts +++ /dev/null @@ -1,95 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -// DEPRECATED!!! DELETE THIS!!! - -import { EventEmitter } from "events"; -import type { State } from "./changefeed"; -import { delay } from "awaiting"; -import { CHANGEFEED_INTEREST_PERIOD_MS } from "@cocalc/nats/sync/synctable"; -import { waitUntilConnected } from "@cocalc/nats/util"; - -export class NatsChangefeed extends EventEmitter { - private client; - private nc; - private query; - private options; - private state: State = "disconnected"; - private natsSynctable?; - - constructor({ client, query, options }: { client; query; options? }) { - super(); - this.client = client; - this.query = query; - this.options = options; - if (this.options != null && this.options.length > 0) { - console.log("NatsChangefeed -- todo: options not implemented", options); - } - } - - connect = async () => { - await waitUntilConnected(); - this.natsSynctable = await this.client.nats_client.changefeed(this.query, { - // atomic=false means less data transfer on changes, but simply does not scale up - // well and is hence quite slow overall. - atomic: true, - immutable: false, - }); - this.state = "connected"; - this.nc = await this.client.nats_client.getConnection(); - this.nc.on?.("reconnect", this.expressInterest); - this.interest(); - this.startWatch(); - const v = this.natsSynctable.get(); - return Object.values(v); - }; - - close = (): void => { - this.nc?.removeListener?.("reconnect", this.expressInterest); - this.natsSynctable?.close(); - this.state = "closed"; - this.emit("close"); // yes "close" not "closed" ;-( - }; - - get_state = (): string => { - return this.state; - }; - - private expressInterest = async () => { - try { - await waitUntilConnected(); - await this.client.nats_client.changefeedInterest(this.query); - } catch (err) { - console.log(`WARNING: changefeed -- ${err}`, this.query); - } - }; - - private interest = async () => { - while (this.state != "closed") { - await this.expressInterest(); - await delay(CHANGEFEED_INTEREST_PERIOD_MS / 2.1); - } - }; - - private startWatch = () => { - if (this.natsSynctable == null) { - return; - } - this.natsSynctable.on( - "change", - (_, { key, value: new_val, prev: old_val }) => { - let x; - if (new_val == null) { - x = { action: "delete", old_val, key }; - } else if (old_val !== undefined) { - x = { action: "update", new_val, old_val, key }; - } else { - x = { action: "insert", new_val, key }; - } - this.emit("update", x); - }, - ); - }; -} diff --git a/src/packages/sync/table/changefeed-nats2.ts b/src/packages/sync/table/changefeed-nats2.ts deleted file mode 100644 index 6bfe7fbeef..0000000000 --- a/src/packages/sync/table/changefeed-nats2.ts +++ /dev/null @@ -1,177 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -import { EventEmitter } from "events"; -import { changefeed, renew } from "@cocalc/nats/changefeed/client"; -import { delay } from "awaiting"; -import { waitUntilConnected } from "@cocalc/nats/util"; - -const LIFETIME = 60000; -const HEARTBEAT = 15000; -const HEARTBEAT_MISS_THRESH = 7500; - -// this should be significantly shorter than HEARTBEAT. -// if user closes browser and comes back, then this is the time they may have -// to wait for their changefeeds to reconnect, since clock jumps forward... -const HEARTBEAT_CHECK_DELAY = 3000; - -const MAX_CHANGEFEED_LIFETIME = 1000 * 60 * 60 * 8; - -// low level debugging of changefeeds -const LOW_LEVEL_DEBUG = false; -const log = LOW_LEVEL_DEBUG - ? (...args) => { - console.log("changefeed: ", ...args); - } - : (..._args) => {}; - -export class NatsChangefeed extends EventEmitter { - private account_id: string; - private query; - private options; - private state: "disconnected" | "connected" | "closed" = "disconnected"; - private natsSynctable?; - private last_hb = 0; - private id?: string; - private lifetime?: number; - - constructor({ - account_id, - query, - options, - }: { - account_id: string; - query; - options?; - }) { - super(); - this.account_id = account_id; - this.query = query; - this.options = options; - } - - connect = async () => { - log("creating new changefeed", this.query); - if (this.state == "closed") return; - this.natsSynctable = await changefeed({ - account_id: this.account_id, - query: this.query, - options: this.options, - heartbeat: HEARTBEAT, - maxActualLifetime: MAX_CHANGEFEED_LIFETIME, - lifetime: LIFETIME, - }); - // @ts-ignore - if (this.state == "closed") return; - this.last_hb = Date.now(); - this.startHeartbeatMonitor(); - this.state = "connected"; - const { - value: { id, lifetime }, - } = await this.natsSynctable.next(); - this.id = id; - this.lifetime = lifetime; - log("got changefeed", { id, lifetime, query: this.query }); - this.startRenewLoop(); - - // @ts-ignore - while (this.state != "closed") { - const { value } = await this.natsSynctable.next(); - this.last_hb = Date.now(); - if (value) { - this.startWatch(); - return value[Object.keys(value)[0]]; - } - } - }; - - close = (): void => { - if (this.state == "closed") { - return; - } - this.kill(); - this.state = "closed"; - log("firing close event for ", this.query); - this.emit("close"); // yes "close" not "closed" ;-( - }; - - get_state = (): string => { - return this.state; - }; - - private startWatch = async () => { - if (this.natsSynctable == null || this.state == "closed") { - return; - } - try { - for await (const x of this.natsSynctable) { - // @ts-ignore - if (this.state == "closed") { - return; - } - this.last_hb = Date.now(); - if (x) { - log("got message ", this.query, x); - this.emit("update", x); - } else { - log("got heartbeat", this.query); - } - } - } catch { - this.close(); - } - }; - - private startHeartbeatMonitor = async () => { - while (this.state != "closed") { - await delay(HEARTBEAT_CHECK_DELAY); - await waitUntilConnected(); - // @ts-ignore - if (this.state == "closed") { - return; - } - if ( - this.last_hb && - Date.now() - this.last_hb > HEARTBEAT + HEARTBEAT_MISS_THRESH - ) { - log("heartbeat failed", this.query, { - last_hb: this.last_hb, - diff: Date.now() - this.last_hb, - thresh: HEARTBEAT + HEARTBEAT_MISS_THRESH, - }); - this.close(); - return; - } - } - }; - - // try to free resources on the server - private kill = async () => { - if (this.id) { - try { - await renew({ - account_id: this.account_id, - id: this.id, - lifetime: -1, - }); - } catch {} - } - }; - - private startRenewLoop = async () => { - while (this.state != "closed" && this.lifetime && this.id) { - // max to avoid weird situation bombarding server or infinite loop - await delay(Math.max(7500, this.lifetime / 3)); - log("renewing with lifetime ", this.lifetime, this.query); - try { - await renew({ - account_id: this.account_id, - id: this.id, - lifetime: this.lifetime, - }); - } catch {} - } - }; -} diff --git a/src/packages/sync/table/synctable.ts b/src/packages/sync/table/synctable.ts index 7e83079aba..b3145749e6 100644 --- a/src/packages/sync/table/synctable.ts +++ b/src/packages/sync/table/synctable.ts @@ -17,9 +17,10 @@ ways of orchestrating a SyncTable. // info about every get/set let DEBUG: boolean = false; -// enable experimental nats database backed changefeed. -// for this to work you must explicitly run the server in @cocalc/database/nats/changefeeds -const USE_NATS = true && !process.env.COCALC_TEST_MODE; +// enable default conat database backed changefeed. +// for this to work you must explicitly run the server in @cocalc/database/conat/changefeeds +// We only turn this off for a mock testing mode. +const USE_CONAT = true && !process.env.COCALC_TEST_MODE; export function set_debug(x: boolean): void { DEBUG = x; @@ -38,7 +39,7 @@ import * as schema from "@cocalc/util/schema"; import mergeDeep from "@cocalc/util/immutable-deep-merge"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { Changefeed } from "./changefeed"; -import { NatsChangefeed } from "./changefeed-nats2"; +import { ConatChangefeed } from "./changefeed-conat"; import { parse_query, to_key } from "./util"; import { isTestClient } from "@cocalc/sync/editor/generic/util"; @@ -67,7 +68,7 @@ function is_fatal(err: string): boolean { export type State = "disconnected" | "connected" | "closed"; export class SyncTable extends EventEmitter { - private changefeed?: Changefeed | NatsChangefeed; + private changefeed?: Changefeed | ConatChangefeed; private query: Query; private client_query: any; private primary_keys: string[]; @@ -655,7 +656,7 @@ export class SyncTable extends EventEmitter { } } - private async connect(): Promise { + private connect = async (): Promise => { const dbg = this.dbg("connect"); dbg(); this.assert_not_closed("connect"); @@ -678,7 +679,7 @@ export class SyncTable extends EventEmitter { await this.create_changefeed(); dbg("connect should have succeeded"); - } + }; private async create_changefeed(): Promise { const dbg = this.dbg("create_changefeed"); @@ -722,15 +723,17 @@ export class SyncTable extends EventEmitter { private create_changefeed_connection = async (): Promise => { let delay_ms: number = 3000; + let warned = false; + let first = true; while (true) { this.close_changefeed(); if ( - USE_NATS && + USE_CONAT && !isTestClient(this.client) && this.client.is_browser() && !this.project_id ) { - this.changefeed = new NatsChangefeed({ + this.changefeed = new ConatChangefeed({ account_id: this.client.client_id?.()!, query: this.query, options: this.options, @@ -745,9 +748,13 @@ export class SyncTable extends EventEmitter { } try { const initval = await this.changefeed.connect(); + if (this.changefeed.get_state() == "closed" || !initval) { throw Error("closed during creation"); } + if (warned) { + console.log(`SUCCESS creating ${this.table} changefeed`); + } return initval; } catch (err) { if (is_fatal(err.toString())) { @@ -755,11 +762,27 @@ export class SyncTable extends EventEmitter { this.close(true); throw err; } - // This can happen because we might suddenly NOT be ready - // to query db immediately after we are ready... - console.log( - `WARNING: ${this.table} -- failed to create changefeed connection; will retry in ${delay_ms}ms -- ${err}`, - ); + if (err.code == 429) { + const message = `${err}`; + console.log(message); + this.client.alert_message?.({ + title: `Too Many Requests (${this.table})`, + message, + type: "error", + }); + await delay(30 * 1000); + } + if (first) { + // don't warn the first time + first = false; + } else { + // This can happen because we might suddenly NOT be ready + // to query db immediately after we are ready... + warned = true; + console.log( + `WARNING: ${this.table} -- failed to create changefeed connection; will retry in ${delay_ms}ms -- ${err}`, + ); + } await delay(delay_ms); delay_ms = Math.min(20000, delay_ms * 1.25); } @@ -802,7 +825,7 @@ export class SyncTable extends EventEmitter { } // awkward code due to typescript weirdness using both - // NatsChangefeed and Changefeed types (for unit testing). + // ConatChangefeed and Changefeed types (for unit testing). private init_changefeed_handlers(): void { const c = this.changefeed as EventEmitter | null; if (c == null) return; diff --git a/src/packages/sync/table/util.ts b/src/packages/sync/table/util.ts index 692faa0565..6815f3052d 100644 --- a/src/packages/sync/table/util.ts +++ b/src/packages/sync/table/util.ts @@ -37,6 +37,18 @@ export function parse_query(query) { } } +export function parseQueryWithOptions(query, options) { + query = parse_query(query); + const table = keys(query)[0]; + const obj = options?.obj; + if (obj != null) { + for (const k in obj) { + query[table][0][k] = obj[k]; + } + } + return { query, table }; +} + const json_stable_stringify = require("json-stable-stringify"); export function to_key(x): string | undefined { if (x === undefined) { diff --git a/src/packages/sync/tsconfig.json b/src/packages/sync/tsconfig.json index 4cdbe9694e..0bdfd8b3a4 100644 --- a/src/packages/sync/tsconfig.json +++ b/src/packages/sync/tsconfig.json @@ -5,5 +5,5 @@ "outDir": "dist" }, "exclude": ["node_modules", "dist", "test"], - "references": [{ "path": "../util" }, { "path": "../nats" }] + "references": [{ "path": "../util" }, { "path": "../conat" }] } diff --git a/src/packages/terminal/index.ts b/src/packages/terminal/index.ts deleted file mode 100644 index 8b708281c3..0000000000 --- a/src/packages/terminal/index.ts +++ /dev/null @@ -1,84 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -/* -Terminal server -*/ - -import { getLogger } from "@cocalc/backend/logger"; -import type { Options, PrimusChannel, PrimusWithChannels } from "./lib/types"; -export { PrimusChannel, PrimusWithChannels }; -import { getChannelName, getRemotePtyChannelName } from "./lib/util"; -import { Terminal } from "./lib/terminal"; -export { RemoteTerminal } from "./lib/remote-terminal"; -export { getRemotePtyChannelName }; -import { EventEmitter } from "events"; - -const logger = getLogger("terminal:index"); - -const terminals: { [name: string]: Terminal } = {}; - -class Terminals extends EventEmitter { - private paths: { [path: string]: true } = {}; - - getOpenPaths = (): string[] => Object.keys(this.paths); - - isOpen = (path) => this.paths[path] != null; - - add = (path: string) => { - this.emit("open", path); - this.paths[path] = true; - }; - - // not used YET: - close = (path: string) => { - this.emit("close", path); - delete this.paths[path]; - }; -} - -export const terminalTracker = new Terminals(); - -// this is used to know which path belongs to which terminal -// (this is the overall tab, not the individual frame -- it's -// used for the processes page) -export function pidToPath(pid: number): string | undefined { - for (const terminal of Object.values(terminals)) { - if (terminal.getPid() == pid) { - return terminal.getPath(); - } - } -} - -// INPUT: primus and description of a terminal session (the path) -// OUTPUT: the name of a websocket channel that serves that terminal session. -export async function terminal( - primus: PrimusWithChannels, - path: string, - options: Options, -): Promise { - const name = getChannelName(path); - if (terminals[name] != null) { - if ( - options.command != null && - options.command != terminals[name].getCommand() - ) { - logger.debug( - "changing command/args for existing terminal and restarting", - path, - ); - terminals[name].setCommand(options.command, options.args); - } - return name; - } - - logger.debug("creating terminal for ", { path }); - const terminal = new Terminal(primus, path, options); - terminals[name] = terminal; - await terminal.init(); - terminalTracker.add(path); - - return name; -} diff --git a/src/packages/terminal/jest.config.js b/src/packages/terminal/jest.config.js deleted file mode 100644 index 140b9467f2..0000000000 --- a/src/packages/terminal/jest.config.js +++ /dev/null @@ -1,6 +0,0 @@ -/** @type {import('ts-jest').JestConfigWithTsJest} */ -module.exports = { - preset: 'ts-jest', - testEnvironment: 'node', - testMatch: ['**/?(*.)+(spec|test).ts?(x)'], -}; diff --git a/src/packages/terminal/lib/remote-pty.test.ts b/src/packages/terminal/lib/remote-pty.test.ts deleted file mode 100644 index aba6d6e846..0000000000 --- a/src/packages/terminal/lib/remote-pty.test.ts +++ /dev/null @@ -1,230 +0,0 @@ -/* - -NOTE: these tests are pretty low level. We don't actually use a websocket at all, -and explicitly shuffle messages around. - ---- -*/ - -import { Terminal } from "./terminal"; -import { - getOpts, - getPrimusMock, - isPidRunning, - waitForPidToChange, -} from "./support"; -import { getRemotePtyChannelName, getChannelName } from "./util"; - -describe("tests remotePty connecting and handling data with **simulated** pty and explicitly pushing messages back and forth (for low level tests)", () => { - let terminal; - const { path, options } = getOpts(); - const primus = getPrimusMock(); - const channel = primus.channel(getChannelName(path)); - const ptyChannel = primus.channel(getRemotePtyChannelName(path)); - - beforeAll(() => { - terminal = new Terminal(primus, path, options); - }); - - afterAll(() => { - terminal.close(); - }); - - it("initialize the terminal", async () => { - await terminal.init(); - expect(typeof terminal.getPid()).toBe("number"); - }); - - let spark1, spark2; - it("create two clients connection to the terminal", async () => { - spark1 = channel.createSpark("192.168.2.1"); - spark2 = channel.createSpark("192.168.2.2"); - for (const s of [spark1, spark2]) { - // it initially tells us the current computeServerId, right when we connect. - const mesg1 = await s.waitForMessage(); - expect(mesg1).toEqual({ cmd: "computeServerId", id: 0 }); - const mesg2 = await s.waitForMessage(); - expect(mesg2).toEqual({ cmd: "no-ignore" }); - } - }); - - let remoteSpark; - it("connect to remote pty channel and observe that local terminal process terminates", async () => { - const pid = terminal.getPid(); - remoteSpark = ptyChannel.createSpark("192.168.2.2"); - expect(terminal.getPid()).toBe(undefined); - // check that original process is no longer running. - expect(await isPidRunning(pid)).toBe(false); - }); - - it("send data from spark1 and see that it is received by the remoteSpark, then respond and see that the client sparks both receive the response", async () => { - // reset client data state - spark1.data = spark2.data = ""; - spark1.emit("data", "17+13"); - expect(await remoteSpark.waitForData("17+13")).toBe("17+13"); - remoteSpark.emit("data", "30"); - expect(await spark1.waitForData("30")).toBe("30"); - expect(await spark2.waitForData("30")).toBe("30"); - }); - - it("disconect the remoteSpark and see that a local pty is spawned again", async () => { - remoteSpark.end(); - await waitForPidToChange(terminal, 0); - const pid = terminal.getPid(); - expect(await isPidRunning(pid)).toBe(true); - }); - - it("connect a remote pty again, send a kill command from one of the spark clients, and check that remote pty receives kill command", async () => { - remoteSpark = ptyChannel.createSpark("192.168.2.2"); - expect((await remoteSpark.waitForMessage()).cmd).toBe("init"); - spark1.emit("data", { cmd: "kill" }); - expect(await remoteSpark.waitForMessage()).toEqual({ cmd: "kill" }); - }); - - it("sends a change of commands and args from client and sees remote pty receives that", async () => { - const command = "/usr/bin/python3"; - const args = ["-b"]; - spark1.emit("data", { - cmd: "set_command", - command, - args, - }); - expect(await remoteSpark.waitForMessage()).toEqual({ - cmd: "set_command", - command, - args, - }); - }); - - it("sends a size message from a client, and observes that remote pty receives a size message", async () => { - const rows = 10; - const cols = 50; - const mesg = { cmd: "size", rows, cols }; - spark1.emit("data", mesg); - expect(await remoteSpark.waitForMessage()).toEqual(mesg); - }); - - it("sends a cwd message from a client, then responds to that from the remoteSpark, and finally checks that the client gets it", async () => { - spark1.emit("data", { cmd: "cwd" }); - spark1.messages = []; - // wait for the message to get sent to our remote spark: - expect(await remoteSpark.waitForMessage()).toEqual({ cmd: "cwd" }); - // send back a cwd - remoteSpark.emit("data", { cmd: "cwd", payload: "/home/user" }); - expect(await spark1.waitForMessage()).toEqual({ - cmd: "cwd", - payload: "/home/user", - }); - }); -}); - -// I disabled all of these for now. They are too difficult to maintain since they are so "fake". - -// describe("test remotePty using actual pty", () => { -// let terminal, remote; -// const { path, options } = getOpts(); -// const primus = getPrimusMock(); -// const channel = primus.channel(getChannelName(path)); -// const ptyChannel = primus.channel(getRemotePtyChannelName(path)); - -// beforeAll(async () => { -// await delay(1000); -// terminal = new Terminal(primus, path, options); -// }); - -// afterAll(() => { -// terminal.close(); -// if (remote != null) { -// remote.close(); -// } -// }); - -// it("initialize the terminal", async () => { -// await terminal.init(); -// expect(typeof terminal.getPid()).toBe("number"); -// }); - -// let spark; -// it("create a normal client connected to the terminal", async () => { -// spark = channel.createSpark("192.168.2.1"); -// const mesg = await spark.waitForMessage(); -// expect(mesg).toEqual({ cmd: "no-ignore" }); -// }); - -// let remoteSpark; -// it("create remote terminal, and observe that local terminal process terminates", async () => { -// const pid = terminal.getPid(); -// remoteSpark = ptyChannel.createSpark("192.168.2.2"); -// remote = new RemoteTerminal(remoteSpark); -// expect(terminal.getPid()).toBe(undefined); -// // check that original process is no longer running. -// expect(await isPidRunning(pid)).toBe(false); -// }); - -// it("observe that remote terminal process gets created", async () => { -// // NOTE: we have to explicitly shuffle the messages along, -// // since our spark mock is VERY minimal and is the same object -// // for both the client and the server. - -// const mesg = await remoteSpark.waitForMessage(); -// remote.handleData(mesg); - -// expect(remote.localPty).not.toEqual(undefined); -// const pid = remote.localPty.pid; -// expect(await isPidRunning(pid)).toBe(true); -// }); - -// it("use bash to compute something", async () => { -// const input = "expr 5077 \\* 389\n"; -// const output = `${5077 * 389}`; -// spark.emit("data", input); - -// // shuttle the data along: -// const data = await remoteSpark.waitForData(input); -// remote.handleData(data); -// const out = await remoteSpark.waitForData(output); -// remoteSpark.emit("data", out); - -// const out2 = await spark.waitForData(output); -// expect(out2).toContain("5077"); -// expect(out2).toContain(output); -// }); - -// it("have client send a size, and see the remote terminal gets that size", async () => { -// spark.emit("data", { cmd: "size", rows: 10, cols: 69 }); -// const mesg = await remoteSpark.waitForMessage(); -// remote.handleData(mesg); -// expect(mesg).toEqual({ cmd: "size", rows: 10, cols: 69 }); -// // now ask the terminal for its size -// spark.emit("data", "stty size\n"); - -// const data = await remoteSpark.waitForData("stty size\n"); -// remote.handleData(data); -// const out = await remoteSpark.waitForData("10 69"); -// remoteSpark.emit("data", out); - -// const out2 = await spark.waitForData("10 69"); -// expect(out2.trim().slice(-5)).toBe("10 69"); -// }); - -// it("tests the cwd command", async () => { -// spark.messages = []; -// // first from browser client to project: -// spark.emit("data", { cmd: "cwd" }); -// const mesg = await remoteSpark.waitForMessage(); -// remote.handleData(mesg); -// const mesg2 = await remoteSpark.waitForMessage(); -// expect(mesg2.payload).toContain("terminal"); -// remoteSpark.emit("data", mesg2); -// const mesg3 = await spark.waitForMessage(); -// expect(mesg3).toEqual(mesg2); -// }); - -// // do this last! -// it("close the RemoteTerminal, and see that a local pty is spawned again", async () => { -// remote.close(); -// await waitForPidToChange(terminal, 0); -// const pid = terminal.getPid(); -// expect(await isPidRunning(pid)).toBe(true); -// }); -// }); diff --git a/src/packages/terminal/lib/remote-terminal.ts b/src/packages/terminal/lib/remote-terminal.ts deleted file mode 100644 index 450919f9b5..0000000000 --- a/src/packages/terminal/lib/remote-terminal.ts +++ /dev/null @@ -1,295 +0,0 @@ -/* -Terminal instance that runs on a remote machine. - -This is a sort of simpler mirror image of terminal.ts. - -This provides a terminal via the "remotePty" mechanism to a project. -The result feels a bit like "ssh'ing to a remote machine", except -the connection comes from the outside over a websocket. When you're -actually using it, though, it's identical to if you ssh out. - -[remote.ts Terminal] ------------> [Project] - -This works in conjunction with src/compute/compute/terminal -*/ - -import getLogger from "@cocalc/backend/logger"; -import { spawn } from "@lydell/node-pty"; -import type { Options, IPty } from "./types"; -import type { Channel } from "@cocalc/comm/websocket/types"; -import { readlink, realpath, writeFile } from "node:fs/promises"; -import { EventEmitter } from "events"; -import { getRemotePtyChannelName } from "./util"; -import { REMOTE_TERMINAL_HEARTBEAT_INTERVAL_MS } from "./terminal"; -import { throttle } from "lodash"; -import { join } from "path"; -import { delay } from "awaiting"; - -// NOTE: shorter than terminal.ts. This is like "2000 lines." -const MAX_HISTORY_LENGTH = 100 * 2000; - -const logger = getLogger("terminal:remote"); - -type State = "init" | "ready" | "closed"; - -export class RemoteTerminal extends EventEmitter { - private state: State = "init"; - private websocket; - private path: string; - private conn: Channel; - private cwd?: string; - private env?: object; - private localPty?: IPty; - private options?: Options; - private size?: { rows: number; cols: number }; - private computeServerId?: number; - private history: string = ""; - private lastData: number = 0; - private healthCheckInterval; - - constructor( - websocket, - path, - { cwd, env }: { cwd?: string; env?: object } = {}, - computeServerId?, - ) { - super(); - this.computeServerId = computeServerId; - this.path = path; - this.websocket = websocket; - this.cwd = cwd; - this.env = env; - logger.debug("create ", { cwd }); - this.connect(); - this.waitUntilHealthy(); - } - - // Why we do this initially is subtle. Basically right when the user opens - // a terminal, the project maybe hasn't set up anything, so there is no - // channel to connect to. The project then configures things, but it doesn't, - // initially see this remote server, which already tried to connect to a channel - // that I guess didn't exist. So we check if we got any response at all, and if - // not we try again, with exponential backoff up to 10s. Once we connect - // and get a response, we switch to about 10s heartbeat checking as usual. - // There is probably a different approach to solve this problem, depending on - // better understanding the async nature of channels, but this does work well. - // Not doing this led to a situation where it always initially took 10.5s - // to connect, which sucks! - private waitUntilHealthy = async () => { - let d = 250; - while (this.state != "closed") { - if (this.isHealthy()) { - this.initRegularHealthChecks(); - return; - } - d = Math.min(10000, d * 1.25); - await delay(d); - } - }; - - private isHealthy = () => { - if (this.state == "closed") { - return true; - } - if ( - Date.now() - this.lastData >= - REMOTE_TERMINAL_HEARTBEAT_INTERVAL_MS + 3000 && - this.websocket.state == "online" - ) { - logger.debug("websocket online but no heartbeat so reconnecting"); - this.reconnect(); - return false; - } - return true; - }; - - private initRegularHealthChecks = () => { - this.healthCheckInterval = setInterval( - this.isHealthy, - REMOTE_TERMINAL_HEARTBEAT_INTERVAL_MS + 3000, - ); - }; - - private reconnect = () => { - logger.debug("reconnect"); - this.conn.removeAllListeners(); - this.conn.end(); - this.connect(); - }; - - private connect = () => { - if (this.state == "closed") { - return; - } - const name = getRemotePtyChannelName(this.path); - logger.debug(this.path, "connect: channel=", name); - this.conn = this.websocket.channel(name); - this.conn.on("data", async (data) => { - // DO NOT LOG EXCEPT FOR VERY LOW LEVEL TEMPORARY DEBUGGING! - // logger.debug(this.path, "channel: data", data); - try { - await this.handleData(data); - } catch (err) { - logger.debug(this.path, "error handling data -- ", err); - } - }); - this.conn.on("end", async () => { - logger.debug(this.path, "channel: end"); - }); - this.conn.on("close", async () => { - logger.debug(this.path, "channel: close"); - this.reconnect(); - }); - if (this.computeServerId != null) { - logger.debug( - this.path, - "connect: sending computeServerId =", - this.computeServerId, - ); - this.conn.write({ cmd: "setComputeServerId", id: this.computeServerId }); - } - }; - - close = () => { - this.state = "closed"; - this.emit("closed"); - this.removeAllListeners(); - this.conn.end(); - if (this.healthCheckInterval) { - clearInterval(this.healthCheckInterval); - } - }; - - private handleData = async (data) => { - if (this.state == "closed") return; - this.lastData = Date.now(); - if (typeof data == "string") { - if (this.localPty != null) { - this.localPty.write(data); - } else { - logger.debug("no pty active, but got data, so let's spawn one locally"); - const pty = await this.initLocalPty(); - if (pty != null) { - // we delete first character since it is the "any key" - // user hit to get terminal going. - pty.write(data.slice(1)); - } - } - } else { - // console.log("COMMAND", data); - switch (data.cmd) { - case "init": - this.options = data.options; - this.size = data.size; - await this.initLocalPty(); - logger.debug("sending history of length", this.history.length); - this.conn.write(this.history); - break; - - case "size": - if (this.localPty != null) { - this.localPty.resize(data.cols, data.rows); - } - break; - - case "cwd": - await this.sendCurrentWorkingDirectoryLocalPty(); - break; - - case undefined: - // logger.debug("received empty data (heartbeat)"); - break; - } - } - }; - - private initLocalPty = async () => { - if (this.state == "closed") return; - if (this.options == null) { - return; - } - if (this.localPty != null) { - return; - } - const command = this.options.command ?? "/bin/bash"; - const args = this.options.args ?? []; - const cwd = this.cwd ?? this.options.cwd; - logger.debug("initLocalPty: spawn -- ", { - command, - args, - cwd, - size: this.size ? this.size : "size not defined", - }); - - const localPty = spawn(command, args, { - cwd, - env: { ...this.options.env, ...this.env }, - rows: this.size?.rows, - cols: this.size?.cols, - }) as IPty; - this.state = "ready"; - logger.debug("initLocalPty: pid=", localPty.pid); - - localPty.onExit(() => { - delete this.localPty; // no longer valid - this.conn.write({ cmd: "exit" }); - }); - - this.localPty = localPty; - if (this.size) { - this.localPty.resize(this.size.cols, this.size.rows); - } - - localPty.onData((data) => { - this.conn.write(data); - - this.history += data; - const n = this.history.length; - if (n >= MAX_HISTORY_LENGTH) { - logger.debug("terminal data -- truncating"); - this.history = this.history.slice(n - MAX_HISTORY_LENGTH / 2); - } - this.saveHistoryToDisk(); - }); - - // set the prompt to show the remote hostname explicitly, - // then clear the screen. - if (command == "/bin/bash") { - this.localPty.write('PS1="(\\h) \\w$ ";reset;history -d $(history 1)\n'); - // alternative -- this.localPty.write('PS1="(\\h) \\w$ "\n'); - } - - return this.localPty; - }; - - private getHome = () => { - return this.env?.["HOME"] ?? process.env.HOME ?? "/home/user"; - }; - - private sendCurrentWorkingDirectoryLocalPty = async () => { - if (this.localPty == null) { - return; - } - // we reply with the current working directory of the underlying - // terminal process, which is why we use readlink and proc below. - const pid = this.localPty.pid; - const home = await realpath(this.getHome()); - const cwd = await readlink(`/proc/${pid}/cwd`); - const path = cwd.startsWith(home) ? cwd.slice(home.length + 1) : cwd; - logger.debug("terminal cwd sent back", { path }); - this.conn.write({ cmd: "cwd", payload: path }); - }; - - private saveHistoryToDisk = throttle(async () => { - const target = join(this.getHome(), this.path); - try { - await writeFile(target, this.history); - } catch (err) { - logger.debug( - `WARNING: failed to save terminal history to '${target}'`, - err, - ); - } - }, 15000); -} - diff --git a/src/packages/terminal/lib/support.ts b/src/packages/terminal/lib/support.ts deleted file mode 100644 index 02b3dc4172..0000000000 --- a/src/packages/terminal/lib/support.ts +++ /dev/null @@ -1,173 +0,0 @@ -/* -Some mocking and other funtionality that is very useful for unit testing. - -The code below does not use Primus *at all* to implement anything -- it's just -a lightweight mocking of the same thing in process for unit testing purposes. -*/ - -import type { PrimusWithChannels } from "./types"; -import { EventEmitter } from "events"; -import { callback, delay } from "awaiting"; -import type { Spark } from "primus"; -import { uuid } from "@cocalc/util/misc"; -import { exec } from "child_process"; -import { once } from "@cocalc/util/async-utils"; - -import debug from "debug"; -const logger = debug("cocalc:test:terminal"); - -const exec1 = (cmd: string, cb) => { - exec(cmd, (_err, stdout, stderr) => { - cb(undefined, { stdout, stderr }); - }); -}; - -export const isPidRunning = async (pid: number) => { - const { stdout } = await callback(exec1, `ps -p ${pid} -o pid=`); - return stdout.trim() != ""; -}; - -export const getCommandLine = async (pid) => { - const { stdout } = await callback(exec1, `ps -p ${pid} -o comm=`); - return stdout; -}; - -export const waitForPidToChange = async (terminal, pid) => { - let i = 1; - while (true) { - const newPid = terminal.getPid(); - if (newPid != null && newPid != pid) { - return newPid; - } - await delay(5 * i); - i += 1; - } -}; - -class PrimusSparkMock extends EventEmitter { - id: string = uuid(); - address: { ip: string }; - data: string = ""; - messages: object[] = []; - - constructor(ip: string) { - super(); - this.address = { ip }; - } - - write = (data) => { - if (this.messages == null) return; - logger("spark write", data); - if (typeof data == "object") { - this.messages.push(data); - } else { - this.data += data; - } - this.emit("write"); - }; - - end = () => { - this.emit("end"); - this.removeAllListeners(); - const t = this as any; - delete t.id; - delete t.address; - delete t.data; - delete t.messages; - }; - - waitForMessage = async () => { - while (true) { - if (this.messages.length > 0) { - return this.messages.shift(); - } - await once(this, "write"); - } - }; - - waitForData = async (x: number | string) => { - let data = ""; - const isDone = () => { - if (typeof x == "number") { - return data.length >= x; - } else { - return data.includes(x); - } - }; - while (!isDone()) { - if (this.data.length > 0) { - data += this.data; - // console.log("so far", { data }); - this.data = ""; - } - if (!isDone()) { - await once(this, "write"); - } - } - return data; - }; -} - -class PrimusChannelMock extends EventEmitter { - name: string; - sparks: { [id: string]: Spark } = {}; - - constructor(name) { - super(); - this.name = name; - } - - write = (data) => { - if (this.sparks == null) return; - for (const spark of Object.values(this.sparks)) { - spark.write(data); - } - }; - - createSpark = (address) => { - const spark = new PrimusSparkMock(address) as unknown as Spark; - this.sparks[spark.id] = spark; - this.emit("connection", spark); - this.on("end", () => { - delete this.sparks[spark.id]; - }); - return spark; - }; - - destroy = () => { - this.removeAllListeners(); - if (this.sparks != null) { - for (const spark of Object.values(this.sparks)) { - spark.end(); - } - } - const t = this as any; - delete t.name; - delete t.sparks; - }; -} - -class PrimusMock { - channels: { [name: string]: PrimusChannelMock } = {}; - - channel = (name) => { - if (this.channels[name] == null) { - this.channels[name] = new PrimusChannelMock(name); - } - return this.channels[name]; - }; -} - -export function getPrimusMock(): PrimusWithChannels { - const primus = new PrimusMock(); - return primus as unknown as PrimusWithChannels; -} - -export function getOpts() { - const name = uuid(); - const path = `.${name}.term-0.term`; - const options = { - path: `${name}.term`, - }; - return { path, options }; -} diff --git a/src/packages/terminal/lib/terminal.test.ts b/src/packages/terminal/lib/terminal.test.ts deleted file mode 100644 index b3321d3d60..0000000000 --- a/src/packages/terminal/lib/terminal.test.ts +++ /dev/null @@ -1,212 +0,0 @@ -import { Terminal } from "./terminal"; -import { - getOpts, - getPrimusMock, - isPidRunning, - getCommandLine, - waitForPidToChange, -} from "./support"; -import { getChannelName } from "./util"; - -describe("very basic test of creating a terminal and changing shell", () => { - let terminal; - const { path, options } = getOpts(); - - beforeAll(() => { - const primus = getPrimusMock(); - terminal = new Terminal(primus, path, options); - }); - - afterAll(() => { - terminal.close(); - }); - - it("checks conditions of terminal before it is initialized", () => { - expect(terminal.getPid()).toBe(undefined); - expect(terminal.getPath()).toBe(options.path); - expect(terminal.getCommand()).toBe("/bin/bash"); - }); - - it("initializes the terminal and checks conditions", async () => { - await terminal.init(); - expect(typeof terminal.getPid()).toBe("number"); - }); - - it("changes the shell to /bin/sh and sees that the pid changes", async () => { - const pid = terminal.getPid(); - terminal.setCommand("/bin/sh", []); - const newPid = await waitForPidToChange(terminal, pid); - expect(pid).not.toBe(newPid); - // check that original process is no longer running. - expect(await isPidRunning(pid)).toBe(false); - }); -}); - -describe("create a shell, connect a client, and communicate with it", () => { - let terminal; - const { path, options } = getOpts(); - const primus = getPrimusMock(); - const channel = primus.channel(getChannelName(path)); - - beforeAll(() => { - terminal = new Terminal(primus, path, options); - }); - - afterAll(() => { - terminal.close(); - }); - - it("initialize the terminal", async () => { - await terminal.init(); - expect(typeof terminal.getPid()).toBe("number"); - }); - - let spark; - it("create a client connection to the terminal", () => { - spark = channel.createSpark("192.168.2.1"); - }); - - it("waits to receive no-ignore command", async () => { - expect(await spark.waitForMessage()).toEqual({ - cmd: "computeServerId", - id: 0, - }); - expect(await spark.waitForMessage()).toEqual({ cmd: "no-ignore" }); - }); - - it("sets the terminal size and confirm it was set", async () => { - const rows = 10, - cols = 100; - expect(terminal.client_sizes[spark.id]).toEqual(undefined); - spark.emit("data", { cmd: "size", rows, cols }); - expect(terminal.client_sizes[spark.id]).toEqual({ rows, cols }); - // also confirm receipt of size message - const mesg = await spark.waitForMessage(); - expect(mesg).toEqual({ cmd: "size", rows, cols }); - }); - - it("gets the current working directory via a command", async () => { - spark.emit("data", { cmd: "cwd" }); - const mesg = await spark.waitForMessage(); - expect(mesg.cmd).toBe("cwd"); - expect(process.cwd().endsWith(mesg.payload)).toBe(true); - }); - - it("write pwd to terminal and get back the current working directory", async () => { - spark.emit("data", "pwd\n"); - spark.data = ""; - const resp = await spark.waitForData(process.cwd()); - expect(resp).toContain(process.cwd()); - }); - - // this test is very flaky and I'm not even sure it makes sense given that the terminal is supposed - // to not autorestart unless you explicilty do the right thing. Disabling it. - // it("send kill command, send some input (to start new shell), and see that pid changes", async () => { - // const pid = terminal.getPid(); - // spark.emit("data", { cmd: "kill" }); - // spark.emit("data", "pwd\n"); - // spark.data = ""; - // const newPid = await waitForPidToChange(terminal, pid); - // expect(pid).not.toBe(newPid); - // expect(await isPidRunning(pid)).toBe(false); - // }); - - it("set shell with set_command see that pid changes", async () => { - const pid = terminal.getPid(); - spark.emit("data", { - cmd: "set_command", - command: "/usr/bin/sleep", - args: ["1000"], - }); - const newPid = await waitForPidToChange(terminal, pid); - expect(pid).not.toBe(newPid); - expect(await isPidRunning(pid)).toBe(false); - expect(await getCommandLine(newPid)).toContain("sleep"); - }); - - it("send some data, then disconnect and reconnect, and verify that history contains that data, and also that terminal continues to work", async () => { - spark.emit("data", "echo 'hello cocalc'\n"); - const resp = await spark.waitForData("hello cocalc"); - expect(resp).toContain("hello cocalc"); - spark.end(); - const id = spark.id; - spark = channel.createSpark("192.168.2.1"); - expect(id).not.toEqual(spark.id); - expect(await spark.waitForMessage()).toEqual({ - cmd: "size", - cols: 100, - rows: 10, - }); - expect(await spark.waitForMessage()).toEqual({ - cmd: "computeServerId", - id: 0, - }); - expect(await spark.waitForMessage()).toEqual({ cmd: "no-ignore" }); - expect(spark.data).toContain("hello cocalc"); - spark.data = ""; - }); -}); - -describe("collaboration -- two clients connected to the same terminal session", () => { - let terminal; - const { path, options } = getOpts(); - const primus = getPrimusMock(); - const channel = primus.channel(getChannelName(path)); - - beforeAll(() => { - terminal = new Terminal(primus, path, options); - }); - - afterAll(() => { - terminal.close(); - }); - - let spark1, spark2; - it("create two clients connection to the terminal", async () => { - await terminal.init(); - spark1 = channel.createSpark("192.168.2.1"); - spark2 = channel.createSpark("192.168.2.2"); - for (const s of [spark1, spark2]) { - expect(await s.waitForMessage()).toEqual({ - cmd: "computeServerId", - id: 0, - }); - const mesg = await s.waitForMessage(); - expect(mesg).toEqual({ cmd: "no-ignore" }); - } - }); - - it("have one client send something, and both clients see the input and result", async () => { - const input = "expr 5077 \\* 389\n"; - const output = `${5077 * 389}`; - spark1.emit("data", input); - const out1 = await spark1.waitForData(output); - expect(out1).toContain("5077"); - expect(out1).toContain(output); - const out2 = await spark2.waitForData(output); - expect(out2).toContain("5077"); - expect(out2).toContain(output); - // also check that output only appears once: - let i = out2.indexOf(output); - expect(out2.indexOf(output, i + 1)).toBe(-1); - }); - - it("set the sizes of the two clients; verify that the min size is returned", async () => { - const rows1 = 15, - cols1 = 90; - const rows2 = 20, - cols2 = 70; - spark1.emit("data", { cmd: "size", rows: rows1, cols: cols1 }); - const mesg1 = await spark1.waitForMessage(); - expect(mesg1).toEqual({ cmd: "size", rows: rows1, cols: cols1 }); - const mesg1a = await spark2.waitForMessage(); - expect(mesg1a).toEqual({ cmd: "size", rows: rows1, cols: cols1 }); - spark2.emit("data", { cmd: "size", rows: rows2, cols: cols2 }); - const mesg2 = await spark2.waitForMessage(); - expect(mesg2).toEqual({ - cmd: "size", - rows: Math.min(rows1, rows2), - cols: Math.min(cols1, cols2), - }); - }); -}); diff --git a/src/packages/terminal/lib/terminal.ts b/src/packages/terminal/lib/terminal.ts deleted file mode 100644 index 39c4c04151..0000000000 --- a/src/packages/terminal/lib/terminal.ts +++ /dev/null @@ -1,692 +0,0 @@ -import type { - ClientCommand, - IPty, - PrimusChannel, - PrimusWithChannels, - Options, -} from "./types"; -import { getChannelName, getRemotePtyChannelName } from "./util"; -import { console_init_filename, len, path_split } from "@cocalc/util/misc"; -import { getLogger } from "@cocalc/backend/logger"; -import { envForSpawn } from "@cocalc/backend/misc"; -import { getCWD } from "./util"; -import { readlink, realpath, readFile, writeFile } from "node:fs/promises"; -import { spawn } from "@lydell/node-pty"; -import { throttle } from "lodash"; -import { exists } from "@cocalc/backend/misc/async-utils-node"; -import { isEqual } from "lodash"; -import type { Spark } from "primus"; -import { join } from "path"; - -const logger = getLogger("terminal:terminal"); - -const CHECK_INTERVAL_MS = 5 * 1000; -export const MAX_HISTORY_LENGTH = 1000 * 1000; -const TRUNCATE_THRESH_MS = 500; -const INFINITY = 999999; -const DEFAULT_COMMAND = "/bin/bash"; - -const EXIT_MESSAGE = "\r\n\r\n[Process completed - press any key]\r\n\r\n"; - -export const REMOTE_TERMINAL_HEARTBEAT_INTERVAL_MS = 7.5 * 1000; - -type MessagesState = "none" | "reading"; -type State = "init" | "ready" | "closed"; - -export class Terminal { - private state: State = "init"; - private options: Options; - private channel: PrimusChannel; - private remotePtyChannel: PrimusChannel; - private history: string = ""; - private path: string; - private client_sizes = {}; - private last_truncate_time: number = Date.now(); - private truncating: number = 0; - private size?: { rows: number; cols: number }; - private backendMessagesBuffer = ""; - private backendMessagesState: MessagesState = "none"; - // two different ways of providing the backend support -- local or remote - private localPty?: IPty; - private remotePty?: Spark; - private computeServerId: number = 0; - private remotePtyHeartbeatInterval; - - constructor(primus: PrimusWithChannels, path: string, options: Options = {}) { - this.options = { command: DEFAULT_COMMAND, ...options }; - this.path = path; - this.channel = primus.channel(getChannelName(path)); - this.channel.on("connection", this.handleClientConnection); - this.remotePtyChannel = primus.channel(getRemotePtyChannelName(path)); - this.remotePtyChannel.on("connection", (conn) => { - logger.debug("new remote terminal connection"); - this.handleRemotePtyConnection(conn); - }); - this.remotePtyHeartbeatInterval = setInterval(() => { - // we always do this (basically a no-op) even if there - // is no remote pty. - this.remotePty?.write({}); - }, REMOTE_TERMINAL_HEARTBEAT_INTERVAL_MS); - } - - init = async () => { - await this.initLocalPty(); - }; - - private initLocalPty = async () => { - if (this.state == "closed") { - throw Error("terminal is closed"); - } - const dbg = (...args) => { - logger.debug("initLocalPty: ", ...args); - }; - if (this.remotePty != null) { - dbg("don't init local pty since there is a remote one."); - return; - } - if (this.localPty != null) { - dbg("don't init local pty since there is already a local one."); - return; - } - - const args: string[] = []; - - const { options } = this; - if (options.args != null) { - for (const arg of options.args) { - if (typeof arg === "string") { - args.push(arg); - } else { - dbg("WARNING -- discarding invalid non-string arg ", arg); - } - } - } else { - const initFilename: string = console_init_filename(this.path); - if (await exists(initFilename)) { - args.push("--init-file"); - args.push(path_split(initFilename).tail); - } - } - if (this.remotePty) { - // switched to a different remote so don't finish initializing a local one - // (we check after each async call) - return; - } - - const { head: pathHead, tail: pathTail } = path_split(this.path); - const env = { - COCALC_TERMINAL_FILENAME: pathTail, - ...envForSpawn(), - ...options.env, - }; - if (env["TMUX"]) { - // If TMUX was set for some reason in the environment that setup - // a cocalc project (e.g., start hub in dev mode from tmux), then - // TMUX is set even though terminal hasn't started tmux yet, which - // confuses our open command. So we explicitly unset it here. - // https://unix.stackexchange.com/questions/10689/how-can-i-tell-if-im-in-a-tmux-session-from-a-bash-script - delete env["TMUX"]; - } - - const { command } = options; - if (command == null) { - throw Error("bug"); - } - const cwd = getCWD(pathHead, options.cwd); - - try { - this.history = (await readFile(this.path)).toString(); - } catch (err) { - dbg("WARNING: failed to load", this.path, err); - } - if (this.remotePty) { - // switched to a different remote, so don't finish initializing a local one - return; - } - - this.setComputeServerId(0); - dbg("spawn", { - command, - args, - cwd, - size: this.size ? this.size : "size not defined", - }); - const localPty = spawn(command, args, { - cwd, - env, - rows: this.size?.rows, - cols: this.size?.cols, - }) as IPty; - dbg("pid=", localPty.pid, { command, args }); - this.localPty = localPty; - - localPty.onData(this.handleDataFromTerminal); - localPty.onExit(async (exitInfo) => { - dbg("exited with code ", exitInfo); - this.handleDataFromTerminal(EXIT_MESSAGE); - delete this.localPty; - }); - // if (command == "/bin/bash") { - // localPty.write("\nreset;history -d $(history 1)\n"); - // } - this.state = "ready"; - return localPty; - }; - - close = () => { - logger.debug("close"); - if ((this.state as State) == "closed") { - return; - } - this.state = "closed"; - this.killPty(); - this.localPty?.destroy(); - this.channel.destroy(); - this.remotePtyChannel.destroy(); - clearInterval(this.remotePtyHeartbeatInterval); - delete this.localPty; - delete this.remotePty; - }; - - getPid = (): number | undefined => { - return this.localPty?.pid; - }; - - // original path - getPath = () => { - return this.options.path; - }; - - getCommand = () => { - return this.options.command; - }; - - setCommand = (command: string, args?: string[]) => { - if (this.state == "closed") return; - if (command == this.options.command && isEqual(args, this.options.args)) { - logger.debug("setCommand: no actual change."); - return; - } - logger.debug( - "setCommand", - { command: this.options.command, args: this.options.args }, - "-->", - { command, args }, - ); - // we track change - this.options.command = command; - this.options.args = args; - if (this.remotePty != null) { - // remote pty - this.remotePty.write({ cmd: "set_command", command, args }); - } else if (this.localPty != null) { - this.localPty.onExit(() => { - this.initLocalPty(); - }); - this.killLocalPty(); - } - }; - - private killPty = () => { - if (this.localPty != null) { - this.killLocalPty(); - } else if (this.remotePty != null) { - this.killRemotePty(); - } - }; - - private killLocalPty = () => { - if (this.localPty == null) return; - logger.debug("killing ", this.localPty.pid); - this.localPty.kill("SIGKILL"); - this.localPty.destroy(); - delete this.localPty; - }; - - private killRemotePty = () => { - if (this.remotePty == null) return; - this.remotePty.write({ cmd: "kill" }); - }; - - private setSizePty = (rows: number, cols: number) => { - if (this.localPty != null) { - this.localPty.resize(cols, rows); - } else if (this.remotePty != null) { - this.remotePty.write({ cmd: "size", rows, cols }); - } - }; - - private saveHistoryToDisk = throttle(async () => { - const target = join(this.getHome(), this.path); - try { - await writeFile(target, this.history); - } catch (err) { - logger.debug( - `WARNING: failed to save terminal history to '${target}'`, - err, - ); - } - }, 15000); - - private resetBackendMessagesBuffer = () => { - this.backendMessagesBuffer = ""; - this.backendMessagesState = "none"; - }; - - private handleDataFromTerminal = (data) => { - //console.log("handleDataFromTerminal", { data }); - if (this.state == "closed") return; - //logger.debug("terminal: term --> browsers", data); - this.handleBackendMessages(data); - this.history += data; - const n = this.history.length; - if (n >= MAX_HISTORY_LENGTH) { - logger.debug("terminal data -- truncating"); - this.history = this.history.slice(n - MAX_HISTORY_LENGTH / 2); - const last = this.last_truncate_time; - const now = Date.now(); - this.last_truncate_time = now; - logger.debug(now, last, now - last, TRUNCATE_THRESH_MS); - if (now - last <= TRUNCATE_THRESH_MS) { - // getting a huge amount of data quickly. - if (!this.truncating) { - this.channel.write({ cmd: "burst" }); - } - this.truncating += data.length; - setTimeout(this.checkIfStillTruncating, CHECK_INTERVAL_MS); - if (this.truncating >= 5 * MAX_HISTORY_LENGTH) { - // only start sending control+c if output has been completely stuck - // being truncated several times in a row -- it has to be a serious non-stop burst... - this.localPty?.write("\u0003"); - } - return; - } else { - this.truncating = 0; - } - } - this.saveHistoryToDisk(); - if (!this.truncating) { - this.channel.write(data); - } - }; - - private checkIfStillTruncating = () => { - if (!this.truncating) { - return; - } - if (Date.now() - this.last_truncate_time >= CHECK_INTERVAL_MS) { - // turn off truncating, and send recent data. - const { truncating, history } = this; - this.channel.write( - history.slice(Math.max(0, history.length - truncating)), - ); - this.truncating = 0; - this.channel.write({ cmd: "no-burst" }); - } else { - setTimeout(this.checkIfStillTruncating, CHECK_INTERVAL_MS); - } - }; - - private handleBackendMessages = (data: string) => { - /* parse out messages like this: - \x1b]49;"valid JSON string here"\x07 - and format and send them via our json channel. - NOTE: such messages also get sent via the - normal channel, but ignored by the client. - */ - if (this.backendMessagesState === "none") { - const i = data.indexOf("\x1b"); - if (i === -1) { - return; // nothing to worry about - } - // stringify it so it is easy to see what is there: - this.backendMessagesState = "reading"; - this.backendMessagesBuffer = data.slice(i); - } else { - this.backendMessagesBuffer += data; - } - if ( - this.backendMessagesBuffer.length >= 5 && - this.backendMessagesBuffer.slice(1, 5) != "]49;" - ) { - this.resetBackendMessagesBuffer(); - return; - } - if (this.backendMessagesBuffer.length >= 6) { - const i = this.backendMessagesBuffer.indexOf("\x07"); - if (i === -1) { - // continue to wait... unless too long - if (this.backendMessagesBuffer.length > 10000) { - this.resetBackendMessagesBuffer(); - } - return; - } - const s = this.backendMessagesBuffer.slice(5, i); - this.resetBackendMessagesBuffer(); - logger.debug( - `handle_backend_message: parsing JSON payload ${JSON.stringify(s)}`, - ); - try { - const payload = JSON.parse(s); - this.channel.write({ cmd: "message", payload }); - } catch (err) { - logger.warn( - `handle_backend_message: error sending JSON payload ${JSON.stringify( - s, - )}, ${err}`, - ); - // Otherwise, ignore... - } - } - }; - - private setSize = (spark: Spark, newSize: { rows; cols }) => { - this.client_sizes[spark.id] = newSize; - try { - this.resize(); - } catch (err) { - // no-op -- can happen if terminal is restarting. - logger.debug("WARNING: resizing terminal", this.path, err); - } - }; - - getSize = (): { rows: number; cols: number } | undefined => { - const sizes = this.client_sizes; - if (len(sizes) == 0) { - return; - } - let rows: number = INFINITY; - let cols: number = INFINITY; - for (const id in sizes) { - if (sizes[id].rows) { - // if, since 0 rows or 0 columns means *ignore*. - rows = Math.min(rows, sizes[id].rows); - } - if (sizes[id].cols) { - cols = Math.min(cols, sizes[id].cols); - } - } - if (rows === INFINITY || cols === INFINITY) { - // no clients with known sizes currently visible - return; - } - // ensure valid values - rows = Math.max(rows ?? 1, rows); - cols = Math.max(cols ?? 1, cols); - // cache for future use. - this.size = { rows, cols }; - return { rows, cols }; - }; - - private resize = () => { - if (this.state == "closed") return; - //logger.debug("resize"); - if (this.localPty == null && this.remotePty == null) { - // nothing to do - return; - } - const size = this.getSize(); - if (size == null) { - return; - } - const { rows, cols } = size; - logger.debug("resize", "new size", rows, cols); - try { - this.setSizePty(rows, cols); - // broadcast out new size to all clients - this.channel.write({ cmd: "size", rows, cols }); - } catch (err) { - logger.debug("terminal channel -- WARNING: unable to resize term", err); - } - }; - - private setComputeServerId = (id: number) => { - this.computeServerId = id; - this.channel.write({ cmd: "computeServerId", id }); - }; - - private sendCurrentWorkingDirectory = async (spark: Spark) => { - if (this.localPty != null) { - await this.sendCurrentWorkingDirectoryLocalPty(spark); - } else if (this.remotePty != null) { - await this.sendCurrentWorkingDirectoryRemotePty(spark); - } - }; - - private getHome = () => { - return process.env.HOME ?? "/home/user"; - }; - - private sendCurrentWorkingDirectoryLocalPty = async (spark: Spark) => { - if (this.localPty == null) { - return; - } - // we reply with the current working directory of the underlying terminal process, - // which is why we use readlink and proc below. - const pid = this.localPty.pid; - // [hsy/dev] wrapping in realpath, because I had the odd case, where the project's - // home included a symlink, hence the "startsWith" below didn't remove the home dir. - const home = await realpath(this.getHome()); - const cwd = await readlink(`/proc/${pid}/cwd`); - // try to send back a relative path, because the webapp does not - // understand absolute paths - const path = cwd.startsWith(home) ? cwd.slice(home.length + 1) : cwd; - logger.debug("terminal cwd sent back", { path }); - spark.write({ cmd: "cwd", payload: path }); - }; - - private sendCurrentWorkingDirectoryRemotePty = async (spark: Spark) => { - if (this.remotePty == null) { - return; - } - // Write cwd command, then wait for a cmd:'cwd' response, and - // forward it to the spark. - this.remotePty.write({ cmd: "cwd" }); - const handle = (mesg) => { - if (typeof mesg == "object" && mesg.cmd == "cwd") { - spark.write(mesg); - this.remotePty?.removeListener("data", handle); - } - }; - this.remotePty.addListener("data", handle); - }; - - private bootAllOtherClients = (spark: Spark) => { - // delete all sizes except this one, so at least kick resets - // the sizes no matter what. - for (const id in this.client_sizes) { - if (id !== spark.id) { - delete this.client_sizes[id]; - } - } - // next tell this client to go fullsize. - if (this.size != null) { - const { rows, cols } = this.size; - if (rows && cols) { - spark.write({ cmd: "size", rows, cols }); - } - } - // broadcast message to all clients telling them to close, but - // telling requestor to ignore. - spark.write({ cmd: "close", ignore: spark.id }); - }; - - private writeToPty = async (data) => { - if (this.state == "closed") return; - // only for VERY low level debugging: - // logger.debug("writeToPty", { data }); - if (this.localPty != null) { - this.localPty.write(data); - } else if (this.remotePty != null) { - this.remotePty.write(data); - } else { - logger.debug("no pty active, but got data, so let's spawn one locally"); - const pty = await this.initLocalPty(); - if (pty != null) { - // we delete first character since it is the "any key" - // user hit to get terminal going. - pty.write(data.slice(1)); - } - } - }; - - private handleDataFromClient = async ( - spark, - data: string | ClientCommand, - ) => { - //logger.debug("terminal: browser --> term", JSON.stringify(data)); - if (typeof data === "string") { - this.writeToPty(data); - } else if (typeof data === "object") { - await this.handleCommandFromClient(spark, data); - } - }; - - private handleCommandFromClient = async ( - spark: Spark, - data: ClientCommand, - ) => { - // control message - //logger.debug("terminal channel control message", JSON.stringify(data)); - if (this.localPty == null && this.remotePty == null) { - await this.initLocalPty(); - } - switch (data.cmd) { - case "size": - this.setSize(spark, { rows: data.rows, cols: data.cols }); - break; - - case "set_command": - this.setCommand(data.command, data.args); - break; - - case "kill": - // send kill signal - this.killPty(); - break; - - case "cwd": - try { - await this.sendCurrentWorkingDirectory(spark); - } catch (err) { - logger.debug( - "WARNING -- issue getting current working directory", - err, - ); - // TODO: the terminal protocol doesn't even have a way - // to report that an error occured, so this silently - // fails. It's just for displaying the current working - // directory, so not too critical. - } - break; - - case "boot": { - this.bootAllOtherClients(spark); - break; - } - } - }; - - private handleClientConnection = (spark: Spark) => { - logger.debug( - this.path, - `new client connection from ${spark.address.ip} -- ${spark.id}`, - ); - - // send current size info - if (this.size != null) { - const { rows, cols } = this.size; - spark.write({ cmd: "size", rows, cols }); - } - - spark.write({ cmd: "computeServerId", id: this.computeServerId }); - - // send burst info - if (this.truncating) { - spark.write({ cmd: "burst" }); - } - - // send history - spark.write(this.history); - - // have history, so do not ignore commands now. - spark.write({ cmd: "no-ignore" }); - - spark.on("end", () => { - if (this.state == "closed") return; - delete this.client_sizes[spark.id]; - this.resize(); - }); - - spark.on("data", async (data) => { - if ((this.state as State) == "closed") return; - try { - await this.handleDataFromClient(spark, data); - } catch (err) { - if (this.state != "closed") { - spark.write(`${err}`); - } - } - }); - }; - - // inform remote pty client of the exact options that are current here. - private initRemotePty = () => { - if (this.remotePty == null) return; - this.remotePty.write({ - cmd: "init", - options: this.options, - size: this.getSize(), - }); - }; - - private handleRemotePtyConnection = (remotePty: Spark) => { - logger.debug( - this.path, - `new pty connection from ${remotePty.address.ip} -- ${remotePty.id}`, - ); - if (this.remotePty != null) { - // already an existing remote connection - // Remove listeners and end it. We have to - // remove listeners or calling end will trigger - // the remotePty.on("end",...) below, which messes - // up everything. - this.remotePty.removeAllListeners(); - this.remotePty.end(); - } - - remotePty.on("end", async () => { - if (this.state == "closed") return; - logger.debug("ending existing remote terminal"); - delete this.remotePty; - await this.initLocalPty(); - }); - - remotePty.on("data", async (data) => { - if ((this.state as State) == "closed") return; - if (typeof data == "string") { - this.handleDataFromTerminal(data); - } else { - if (this.localPty != null) { - // already switched back to local - return; - } - if (typeof data == "object") { - switch (data.cmd) { - case "setComputeServerId": - this.setComputeServerId(data.id); - break; - case "exit": { - this.handleDataFromTerminal(EXIT_MESSAGE); - break; - } - } - } - } - }); - - this.remotePty = remotePty; - this.initRemotePty(); - this.killLocalPty(); - }; -} diff --git a/src/packages/terminal/lib/types.ts b/src/packages/terminal/lib/types.ts deleted file mode 100644 index 55dc42b96a..0000000000 --- a/src/packages/terminal/lib/types.ts +++ /dev/null @@ -1,62 +0,0 @@ -import type { EventEmitter } from "events"; -import Primus from "primus"; -import type { Spark } from "primus"; -import type { IPty as IPty0 } from "@lydell/node-pty"; - -// upstream typings not quite right -export interface IPty extends IPty0 { - destroy: () => void; -} - -export interface Options { - // path -- the "original" path to the terminal, not the derived "term_path" - path?: string; - command?: string; - args?: string[]; - env?: { [key: string]: string }; - // cwd -- if not set, the cwd is directory of "path" - cwd?: string; -} - -export interface PrimusChannel extends EventEmitter { - write: (data: object | string) => void; - destroy: () => void; - // createSpark is not on the real PrimusChannel, but it's part of our mock version for - // unit testing in support.ts - createSpark: (address: string) => Spark; -} - -export interface PrimusWithChannels extends Primus { - channel: (name: string) => PrimusChannel; -} - -interface SizeClientCommand { - cmd: "size"; - rows: number; - cols: number; -} - -interface SetClientCommand { - cmd: "set_command"; - command: string; - args: string[]; -} - -interface KillClientCommand { - cmd: "kill"; -} - -interface CWDClientCommand { - cmd: "cwd"; -} - -interface BootClientCommand { - cmd: "boot"; -} - -export type ClientCommand = - | SizeClientCommand - | SetClientCommand - | KillClientCommand - | CWDClientCommand - | BootClientCommand; diff --git a/src/packages/terminal/lib/util.ts b/src/packages/terminal/lib/util.ts deleted file mode 100644 index c4ca71debd..0000000000 --- a/src/packages/terminal/lib/util.ts +++ /dev/null @@ -1,22 +0,0 @@ -export function getChannelName(path: string): string { - return `terminal:${path}`; -} - -export function getRemotePtyChannelName(path: string): string { - return `terminal-pty:${path}`; -} - -export function getCWD(pathHead, cwd?): string { - // working dir can be set explicitly, and either be an empty string or $HOME - if (cwd != null) { - const HOME = process.env.HOME ?? "/home/user"; - if (cwd === "") { - return HOME; - } else if (cwd.startsWith("$HOME")) { - return cwd.replace("$HOME", HOME); - } else { - return cwd; - } - } - return pathHead; -} diff --git a/src/packages/terminal/package.json b/src/packages/terminal/package.json deleted file mode 100644 index 8f0f200d4f..0000000000 --- a/src/packages/terminal/package.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "name": "@cocalc/terminal", - "version": "0.1.2", - "description": "CoCalc's Nodejs Terminal Server", - "main": "./dist/index.js", - "scripts": { - "preinstall": "npx only-allow pnpm", - "build": "../node_modules/.bin/tsc --build", - "clean": "rm -rf dist node_modules", - "_test_doc_": "--runInBand -- serial only because crashes when running all tests across all packages", - "test": "pnpm exec jest --runInBand --forceExit ", - "tsc": "../node_modules/.bin/tsc --watch --pretty --preserveWatchOutput" - }, - "files": [ - "dist/**", - "bin/**", - "README.md", - "package.json" - ], - "author": "SageMath, Inc.", - "keywords": [ - "cocalc", - "jupyter", - "terminal" - ], - "license": "SEE LICENSE.md", - "dependencies": { - "@cocalc/api-client": "workspace:*", - "@cocalc/backend": "workspace:*", - "@cocalc/comm": "workspace:*", - "@cocalc/primus-multiplex": "^1.1.0", - "@cocalc/primus-responder": "^1.0.5", - "@cocalc/util": "workspace:*", - "@lydell/node-pty": "^1.1.0", - "awaiting": "^3.0.0", - "debug": "^4.4.0", - "lodash": "^4.17.21", - "primus": "^8.0.9" - }, - "homepage": "https://github.com/sagemathinc/cocalc/tree/master/src/packages/terminal", - "repository": { - "type": "git", - "url": "https://github.com/sagemathinc/cocalc" - }, - "devDependencies": { - "@types/lodash": "^4.14.202", - "@types/node": "^18.16.14", - "@types/primus": "^7.3.9" - } -} diff --git a/src/packages/terminal/tsconfig.json b/src/packages/terminal/tsconfig.json deleted file mode 100644 index d039a30854..0000000000 --- a/src/packages/terminal/tsconfig.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "extends": "../tsconfig.json", - "compilerOptions": { - "rootDir": "./", - "outDir": "dist" - }, - "exclude": ["node_modules", "dist", "test"], - "references": [ - { "path": "../api-client" }, - { "path": "../backend" }, - { "path": "../comm" }, - { "path": "../util" } - ] -} diff --git a/src/packages/util/async-utils.ts b/src/packages/util/async-utils.ts index 1520b88d7b..367c81e7f5 100644 --- a/src/packages/util/async-utils.ts +++ b/src/packages/util/async-utils.ts @@ -16,9 +16,53 @@ The two helpful async/await libraries I found are: */ import * as awaiting from "awaiting"; - import { reuseInFlight } from "./reuse-in-flight"; +interface RetryOptions { + start?: number; + decay?: number; + max?: number; + min?: number; + timeout?: number; + log?: (...args) => void; +} + +// loop calling the async function f until it returns true. +// It optionally can take a timeout, which if hit it will +// throw Error('timeout'). retry_until_success below is an +// a variant of this pattern keeps retrying until f doesn't throw. +// The input function f must always return true or false, +// which helps a lot to avoid bugs. +export async function until( + f: (() => Promise) | (() => boolean), + { + start = 500, + decay = 1.3, + max = 15000, + min = 50, + timeout = 0, + log, + }: RetryOptions = {}, +) { + const end = timeout ? Date.now() + timeout : undefined; + let d = start; + while (end === undefined || Date.now() < end) { + const x = await f(); + if (x) { + return; + } + if (end) { + d = Math.max(min, Math.min(end - Date.now(), Math.min(max, d * decay))); + } else { + d = Math.max(min, Math.min(max, d * decay)); + } + log?.(`will retry in ${Math.round(d / 1000)} seconds`); + await awaiting.delay(d); + } + log?.("FAILED: timeout"); + throw Error("timeout"); +} + export { asyncDebounce, asyncThrottle } from "./async-debounce-throttle"; // turns a function of opts, which has a cb input into @@ -123,54 +167,51 @@ import { CB } from "./types/database"; in our applications. If timeout_ms is nonzero and event doesn't happen an exception is thrown. + If the obj throws 'closed' before the event is emitted, + then this throws an error, since clearly event can never be emitted. */ export async function once( obj: EventEmitter, event: string, timeout_ms: number = 0, ): Promise { - if (obj == null) { - throw Error("once -- obj is undefined"); - } - if (typeof obj.once != "function") { + if (obj == null) throw Error("once -- obj is undefined"); + if (typeof obj.once != "function") throw Error("once -- obj.once must be a function"); - } - if (timeout_ms > 0) { - // just to keep both versions more readable... - return once_with_timeout(obj, event, timeout_ms); - } - let val: any[] = []; - function wait(cb: Function): void { - obj.once(event, function (...args): void { - val = args; - cb(); - }); - } - await awaiting.callback(wait); - return val; -} -async function once_with_timeout( - obj: EventEmitter, - event: string, - timeout_ms: number, -): Promise { - let val: any[] = []; - function wait(cb: Function): void { - function fail(): void { - obj.removeListener(event, handler); - cb("timeout"); + return new Promise((resolve, reject) => { + let timer: NodeJS.Timeout | undefined; + + function cleanup() { + obj.removeListener(event, onEvent); + obj.removeListener("closed", onClosed); + if (timer) clearTimeout(timer); + } + + function onEvent(...args: any[]) { + cleanup(); + resolve(args); } - const timer = setTimeout(fail, timeout_ms); - function handler(...args): void { - clearTimeout(timer); - val = args; - cb(); + + function onClosed() { + cleanup(); + reject(new Error(`once: "${event}" not emitted before "closed"`)); } - obj.once(event, handler); - } - await awaiting.callback(wait); - return val; + + function onTimeout() { + cleanup(); + reject( + new Error(`once: timeout of ${timeout_ms}ms waiting for "${event}"`), + ); + } + + obj.once(event, onEvent); + obj.once("closed", onClosed); + + if (timeout_ms > 0) { + timer = setTimeout(onTimeout, timeout_ms); + } + }); } // Alternative to callback_opts that behaves like the callback defined in awaiting. @@ -260,3 +301,28 @@ export async function parallelHandler({ // Wait for all remaining promises to finish await Promise.all(promiseQueue); } + +// use it like this: +// resp = await withTimeout(promise, 3000); +// and if will throw a timeout if promise takes more than 3s to resolve, +// though of course whatever code is running in promise doesn't actually +// get interrupted. +export async function withTimeout(p: Promise, ms: number) { + let afterFired = false; + p.catch((err) => { + if (afterFired) { + console.warn("WARNING: withTimeout promise rejected", err); + } + }); + let to; + return Promise.race([ + p, + new Promise( + (_, reject) => + (to = setTimeout(() => { + afterFired = true; + reject(new Error("timeout")); + }, ms)), + ), + ]).finally(() => clearTimeout(to)); +} diff --git a/src/packages/util/conat.ts b/src/packages/util/conat.ts new file mode 100644 index 0000000000..77739857b4 --- /dev/null +++ b/src/packages/util/conat.ts @@ -0,0 +1,4 @@ +// Some very generic conat related parameters + +// how frequently +export const CONAT_OPEN_FILE_TOUCH_INTERVAL = 30000; diff --git a/src/packages/util/consts/project.ts b/src/packages/util/consts/project.ts index 231cf39f24..eec570e898 100644 --- a/src/packages/util/consts/project.ts +++ b/src/packages/util/consts/project.ts @@ -7,20 +7,11 @@ export const PROJECT_EXEC_DEFAULT_TIMEOUT_S = 60; export const TIMEOUT_CALLING_PROJECT = "timeout"; -const NATS_TIMEOUT_MSG = "NatsError: TIMEOUT"; - export const TIMEOUT_CALLING_PROJECT_MSG = "Timeout communicating with project."; export const IS_TIMEOUT_CALLING_PROJECT = (err) => { - if (err === TIMEOUT_CALLING_PROJECT || err === NATS_TIMEOUT_MSG) { - return true; - } - - if ( - typeof err?.toString === "function" && - err?.toString() === NATS_TIMEOUT_MSG - ) { + if (err === TIMEOUT_CALLING_PROJECT) { return true; } return false; diff --git a/src/packages/util/db-schema/client-db.ts b/src/packages/util/db-schema/client-db.ts index a86719e63d..d26f0f1bcd 100644 --- a/src/packages/util/db-schema/client-db.ts +++ b/src/packages/util/db-schema/client-db.ts @@ -84,48 +84,6 @@ class ClientDB { throw Error("primary key must be a string or array of strings"); } } - - // Given rows (as objects) obtained by querying a table or virtual table, - // converts any non-null string ISO timestamps to Date objects. This is - // needed because we transfer data from the database to the browser using - // JSONCodec (via NATS) and that turns Date objects into ISO timestamp strings. - // This turns them back, but by using the SCHEMA, *not* a heuristic or regexp - // to identify which fields to change. - // NOTE: this *mutates* rows. - processDates = ({ - table, - rows, - }: { - table: string; - rows: object[] | object; - }) => { - let t = SCHEMA[table]; - if (t == null) { - return; - } - if (typeof t.virtual == "string") { - t = SCHEMA[t.virtual]; - } - const timeFields: string[] = []; - const { fields } = t; - for (const field in fields) { - if (fields[field].type == "timestamp") { - timeFields.push(field); - } - } - if (timeFields.length == 0) { - // nothing to do. - return; - } - const v = is_array(rows) ? rows : [rows]; - for (const row of v) { - for (const field of timeFields) { - if (typeof row[field] == "string") { - row[field] = new Date(row[field]); - } - } - } - }; } export const client_db = new ClientDB(); diff --git a/src/packages/util/db-schema/file-use.ts b/src/packages/util/db-schema/file-use.ts index 94ac7a7240..7d02eea658 100644 --- a/src/packages/util/db-schema/file-use.ts +++ b/src/packages/util/db-schema/file-use.ts @@ -92,7 +92,7 @@ Table({ // Otherwise we touch the project just for seeing notifications or opening // the file, which is confusing and wastes a lot of resources. const x = obj.users != null ? obj.users[account_id] : undefined; - // edit/chat/open fields are now strings due to using nats and not auto + // edit/chat/open fields are now strings due to using conat and not auto // autoconverting them to Dates at this point, hence comparing iso strings: const recent = minutes_ago(3).toISOString(); if ( diff --git a/src/packages/util/db-schema/index.ts b/src/packages/util/db-schema/index.ts index fa97a66b85..8fd69895ec 100644 --- a/src/packages/util/db-schema/index.ts +++ b/src/packages/util/db-schema/index.ts @@ -46,10 +46,8 @@ import "./news"; import "./organizations"; import "./password-reset"; import "./pg-system"; -import "./project-info"; import "./project-invite-tokens"; import "./project-log"; -import "./project-status"; import "./projects"; import "./public-path-stars"; import "./public-paths"; diff --git a/src/packages/util/db-schema/project-info.ts b/src/packages/util/db-schema/project-info.ts deleted file mode 100644 index 1962a3884f..0000000000 --- a/src/packages/util/db-schema/project-info.ts +++ /dev/null @@ -1,66 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -/* -This table contains real-time information about a running project. In particular, this includes the running processes, CGroup status, etc. It is updated frequently, hence use it wisely... -*/ - -import { Table } from "./types"; - -Table({ - name: "project_info", - fields: { - project_id: { - type: "uuid", - desc: "The project id.", - }, - info: { - // change this according to all the usual schema rules - type: "map", - pg_type: "JSONB", - desc: "Info about the project", - }, - }, - rules: { - durability: "ephemeral", // won't be stored in the database at all ever. - desc: - "Information about running processes, cgroups, disk space, etc. of projects", - primary_key: ["project_id"], // can list multiple another field if you want to have multiple records for a project. - user_query: { - get: { - pg_where: ["projects"], - fields: { - project_id: null, - info: null, - }, - }, - set: { - // users can set that they are interested in this - fields: { - project_id: "project_id", - info: true, - }, - }, - }, - - project_query: { - get: { - pg_where: [{ "project_id = $::UUID": "project_id" }], - fields: { - project_id: null, - info: null, - }, - }, - set: { - // delete=true, since project *IS* allowed to delete entries - delete: true, - fields: { - project_id: "project_id", - info: true, - }, - }, - }, - }, -}); diff --git a/src/packages/util/db-schema/project-log.ts b/src/packages/util/db-schema/project-log.ts index 01837f5520..e2790b8dd4 100644 --- a/src/packages/util/db-schema/project-log.ts +++ b/src/packages/util/db-schema/project-log.ts @@ -7,6 +7,8 @@ import { deep_copy, uuid } from "../misc"; import { SCHEMA as schema } from "./index"; import { Table } from "./types"; +const DEFAULT_LIMIT = 750; + Table({ name: "project_log", rules: { @@ -22,7 +24,7 @@ Table({ get: { pg_where: ["time >= NOW() - interval '2 months'", "projects"], pg_changefeed: "projects", - options: [{ order_by: "-time" }, { limit: 300 }], + options: [{ order_by: "-time" }, { limit: DEFAULT_LIMIT }], throttle_changes: 2000, fields: { id: null, diff --git a/src/packages/util/db-schema/project-status.ts b/src/packages/util/db-schema/project-status.ts deleted file mode 100644 index 4e3c0ebdb1..0000000000 --- a/src/packages/util/db-schema/project-status.ts +++ /dev/null @@ -1,70 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -/* -This table contains the current overall status about a running project. -This is the sister-table to "project-info". -In contrast, this table provides much less frequently changed pieces of status information. -For example, project version, certain "alerts", disk usage, etc. -Its intended usage is to subscribe to it once you open a project and notify the user if certain alerts go off. -*/ - -import { Table } from "./types"; - -Table({ - name: "project_status", - fields: { - project_id: { - type: "uuid", - desc: "The project id.", - }, - status: { - // change this according to all the usual schema rules - type: "map", - pg_type: "JSONB", - desc: "Status of this project", - }, - }, - rules: { - durability: "ephemeral", // won't be stored in the database at all ever. - desc: - "Project status, like version, certain 'alerts', disk usage, ...", - primary_key: ["project_id"], // can list multiple another field if you want to have multiple records for a project. - user_query: { - get: { - pg_where: ["projects"], - fields: { - project_id: null, - status: null, - }, - }, - set: { - // users can set that they are interested in this - fields: { - project_id: "project_id", - status: true, - }, - }, - }, - - project_query: { - get: { - pg_where: [{ "project_id = $::UUID": "project_id" }], - fields: { - project_id: null, - status: null, - }, - }, - set: { - // delete=true, since project *IS* allowed to delete entries - delete: true, - fields: { - project_id: "project_id", - status: true, - }, - }, - }, - }, -}); diff --git a/src/packages/util/db-schema/projects.ts b/src/packages/util/db-schema/projects.ts index 3512647835..6915307691 100644 --- a/src/packages/util/db-schema/projects.ts +++ b/src/packages/util/db-schema/projects.ts @@ -68,6 +68,8 @@ Table({ run_quota: null, site_license: null, status: null, + // security model is anybody with access to the project should be allowed to know this token. + secret_token: null, state: null, last_edited: null, last_active: null, @@ -330,6 +332,11 @@ Table({ render: { type: "json", editable: false }, }, notes: NOTES, + secret_token: { + type: "string", + pg_type: "VARCHAR(256)", + desc: "Random ephemeral secret token used temporarily by project to authenticate with hub.", + }, }, }); @@ -566,7 +573,6 @@ export interface ProjectStatus { "sage_server.pid"?: number; // pid of sage server process start_ts?: number; // timestamp, when project server started session_id?: string; // unique identifyer - secret_token?: string; // long random secret token that is needed to communicate with local_hub version?: number; // version number of project code disk_MB?: number; // MB of used disk installed?: boolean; // whether code is installed diff --git a/src/packages/util/db-schema/server-settings.ts b/src/packages/util/db-schema/server-settings.ts index 088cfb63e3..2cc885379c 100644 --- a/src/packages/util/db-schema/server-settings.ts +++ b/src/packages/util/db-schema/server-settings.ts @@ -129,4 +129,11 @@ export interface Customize { githubProjectId?: string; support?: string; supportVideoCall?: string; + version?: { + min_project?: number; + min_compute_server?: number; + min_browser?: number; + recommended_browser?: number; + compute_server_min_project?: number; + }; } diff --git a/src/packages/util/db-schema/site-defaults.ts b/src/packages/util/db-schema/site-defaults.ts index 3578fecdd3..7f3c5156b5 100644 --- a/src/packages/util/db-schema/site-defaults.ts +++ b/src/packages/util/db-schema/site-defaults.ts @@ -29,7 +29,7 @@ export const TAGS = [ "Email", "Logo", "Version", - "Nats", + "Conat", "Stripe", "captcha", "Zendesk", @@ -97,6 +97,7 @@ export type SiteSettingsKeys = | "ssh_gateway_fingerprint" | "versions" | "version_min_project" + | "version_min_compute_server" | "version_compute_server_min_project" | "version_min_browser" | "version_recommended_browser" @@ -542,7 +543,15 @@ export const site_settings_conf: SiteSettings = { }, version_min_project: { name: "Required project version", - desc: "Minimal version required by projects (if project older, will be force restarted).", + desc: "Minimal version required by projects (if older, will terminate).", + default: "0", + valid: only_nonneg_int, + show: () => true, + tags: ["Version"], + }, + version_min_compute_server: { + name: "Required compute server version", + desc: "Minimal version required by compute server (if older, will terminate).", default: "0", valid: only_nonneg_int, show: () => true, diff --git a/src/packages/util/db-schema/site-settings-extras.ts b/src/packages/util/db-schema/site-settings-extras.ts index 4aa008362d..7e0b564468 100644 --- a/src/packages/util/db-schema/site-settings-extras.ts +++ b/src/packages/util/db-schema/site-settings-extras.ts @@ -184,14 +184,12 @@ function custom_llm_display(value: string): string { export type SiteSettingsExtrasKeys = | "pii_retention" - | "nats_heading" - | "nats_server" - | "nats_port" - | "nats_ws_port" - | "nats_password" - | "nats_auth_nseed" - | "nats_auth_xseed" - | "nats_project_server" + | "conat_heading" + | "conat_server" + | "conat_password" + | "conat_valkey" + // | "conat_socketio_count" + // | "conat_cluster_port" | "stripe_heading" | "stripe_publishable_key" | "stripe_secret_key" @@ -276,64 +274,49 @@ const DEFAULT_COMPUTE_SERVER_IMAGES_JSON = // not public, but admins can edit them export const EXTRAS: SettingsExtras = { - nats_heading: { - name: "NATS Configuration", - desc: "Configuration of [NATS server](https://nats.io/), which CoCalc uses extensively for communication.", + conat_heading: { + name: "Conat Configuration", + desc: "Conat is a [NATS](https://nats.io/)-like [socketio](https://socket.io/) websocket server and persistence layer that CoCalc uses extensively for communication.", default: "", type: "header", - tags: ["Nats"], + tags: ["Conat"], }, - // Nats config is loaded in packages/server/nats/credentials.ts - nats_server: { - name: "Nats Server", - desc: "Hostname of server where NATS is running. Defaults to localhost or `$COCALC_NATS_SERVER` if not specified here. (TODO: support multiple servers for high availability.)", - default: "localhost", - password: false, - tags: ["Nats"], - }, - nats_port: { - name: "Nats TCP Port", - desc: "Port that NATS is serving on. Defaults to 4222 or `$COCALC_NATS_PORT` if not specified here.", - default: "4222", - password: false, - tags: ["Nats"], - }, - nats_ws_port: { - name: "Nats Websocket Port", - desc: "Port that NATS websocket server is serving on. Defaults to 8443 or `$COCALC_NATS_WS_PORT` if not specified here. This gets proxied to browser clients.", - default: "8443", - password: false, - tags: ["Nats"], - }, - nats_project_server: { - name: "Nats Project Server", - desc: "Name of the NATS server that projects should connect to. This should be either `hostname:port` for TCP or one of `ws://hostname:port` or `wss://hostname:port` for a WebSocket. Do not include the basepath for the websocket address. If not given, the tcp NATS server and port specified above is used.", + // Conat config may be loaded from via code in packages/server/conat/configuration.ts + conat_server: { + name: "Conat Server URL", + desc: "URL of server where Conat is available. Defaults to `$CONAT_SERVER` env variable if that is given. This URL should include the base path. Examples: https://cocalc.com, https://localhoast:4043/3fa218e5-7196-4020-8b30-e2127847cc4f/port/5002/", default: "", password: false, - tags: ["Nats"], + tags: ["Conat"], }, - nats_password: { - name: "Nats Password", - desc: "Password required for nats account configured above on the NATS server. If not given, then the contents of the file `$SECRETS/nats_password` (or `$COCALC_ROOT/data/secrets/nats_password`) is used, if it exists. IMPORTANT: the nseed and xseed secrets must also exist in order for the authentication microservice to communicate with nats-server and authenticate users.", + conat_password: { + name: "Conat Password", + desc: "Password for conat *hub* admin account. If not given, then the contents of the file `$SECRETS/conat_password` (or `$COCALC_ROOT/data/secrets/conat_password`) is used, if it exists.", default: "", password: true, - tags: ["Nats"], - }, - nats_auth_nseed: { - name: "Nats Authentication Callout - Signing Private Key", - desc: "The ed25519 nkeys secret key that is used by the auth callout microservice. If not given, then the contents of the file `$SECRETS/nats_auth_nseed` (or `$COCALC_ROOT/data/secrets/nats_auth_nseed`) is used, if it exists. This is an *account* private nkey used by the server to digitally sign messages to the auth callout service: `nk -gen account`", - default: "", - password: true, - tags: ["Nats"], - }, - nats_auth_xseed: { - name: "Nats Authentication Callout - Encryption Private Key", - desc: "The ed25519 nkeys secret key that is used by the auth callout microservice. If not given, then the contents of the file `$SCRETS/nats_auth_xseed` (or `$COCALC_ROOT/data/secrets/nats_auth_xseed`) is used, if it exists. This is a *curve* private nkey used by the auth callout service to encrypt responses to the server: `nk -gen curve`", + tags: ["Conat"], + }, + // conat_socketio_count: { + // name: "Number of Conat Socketio Servers to Run", + // desc: "The number of conat [Socketio](https://socket.io/) servers to create. When running CoCalc on a single server, you can run a single socketio websocket server in the same nodejs process as everything else. Alternatively, if you set this value to a number $n$ bigger than 1 and enable valkey or the Conat cluster port below, then $n$ separate socket.io servers will be spawned. The main hub server will proxy connections to these servers. This allows you to scale the traffic load beyond a single CPU.", + // default: "1", + // valid: only_pos_int, + // tags: ["Conat"], + // }, + // conat_cluster_port: { + // name: "Conat Socketio Cluster Adapter Port", + // desc: "If set, the [Socketio cluster adapter](https://github.com/socketio/socket.io-cluster-adapter) is used, listening on this port. Set to 0 to disable.", + // default: "0", + // valid: only_nonneg_int, + // tags: ["Conat"], + // }, + conat_valkey: { + name: "Valkey Connection String", + desc: "[Valkey](https://valkey.io/) is required to run multiple Conat socketio servers, which is required to scale to thousands of simultaneous connections. This is the connection URL, which is of the form [valkey://user:password@host:port/dbnum](https://valkey.io/topics/cli/). E.g., `valkey://127.0.0.1:6379`. For HA with sentinels, use something like 'sentinel://valkey-sentinel-0[:port],valkey-sentinel-1[:port],valkey-sentinel-2[:port]' as the connection string instead of 'valkey:// ..'", default: "", - password: true, - tags: ["Nats"], + password: false, + tags: ["Conat"], }, - openai_section: { name: "Language Model Configuration", desc: "", diff --git a/src/packages/util/db-schema/syncstring-schema.ts b/src/packages/util/db-schema/syncstring-schema.ts index faafb5d16a..b8d6b49984 100644 --- a/src/packages/util/db-schema/syncstring-schema.ts +++ b/src/packages/util/db-schema/syncstring-schema.ts @@ -310,7 +310,7 @@ Table({ }, seq_info: { type: "map", - desc: "nats info about snapshot -- {seq:number; prev_seq?:number; index:number}", + desc: "conat-assigned info about snapshot -- {seq:number; prev_seq?:number; index:number}", }, sent: { type: "timestamp", diff --git a/src/packages/util/event-iterator.ts b/src/packages/util/event-iterator.ts new file mode 100644 index 0000000000..c3ca053301 --- /dev/null +++ b/src/packages/util/event-iterator.ts @@ -0,0 +1,272 @@ +/* +LICENSE: MIT + +This is a slight fork of + +https://github.com/sapphiredev/utilities/tree/main/packages/event-iterator + +because upstream is slightly broken and what it actually does doesn't +agree with the docs. I can see why. Upstream would capture ['arg1','arg2']] +for an event emitter doing this + + emitter.emit('foo', 'arg1', 'arg2') + +But for our application we only want 'arg1'. I thus added a map option, +which makes it easy to do what we want. +*/ + +import type { EventEmitter } from "node:events"; + +/** + * A filter for an EventIterator. + */ +export type EventIteratorFilter = (value: V) => boolean; + +/** + * Options to be passed to an EventIterator. + */ +export interface EventIteratorOptions { + /** + * The filter. + */ + filter?: EventIteratorFilter; + + // maps the array of args emitted by the event emitter a V + map?: (args: any[]) => V; + + /** + * The timeout in ms before ending the EventIterator. + */ + idle?: number; + + /** + * The limit of events that pass the filter to iterate. + */ + limit?: number; + + // called when iterator ends -- use to do cleanup. + onEnd?: (iter?: EventIterator) => void; +} + +/** + * An EventIterator, used for asynchronously iterating over received values. + */ +export class EventIterator + implements AsyncIterableIterator +{ + /** + * The emitter to listen to. + */ + public readonly emitter: EventEmitter; + + /** + * The event the event iterator is listening for to receive values from. + */ + public readonly event: string; + + /** + * The filter used to filter out values. + */ + public filter: EventIteratorFilter; + + public map; + + /** + * Whether or not the EventIterator has ended. + */ + #ended = false; + + private onEnd?: (iter?: EventIterator) => void; + + /** + * The amount of idle time in ms before moving on. + */ + readonly #idle?: number; + + /** + * The queue of received values. + */ + #queue: V[] = []; + + private err: any = undefined; + + /** + * The amount of events that have passed the filter. + */ + #passed = 0; + + /** + * The limit before ending the EventIterator. + */ + readonly #limit: number; + + /** + * The timer to track when this will idle out. + */ + #idleTimer: NodeJS.Timeout | undefined | null = null; + + /** + * The push handler with context bound to the instance. + */ + readonly #push: (this: EventIterator, ...value: unknown[]) => void; + + /** + * @param emitter The event emitter to listen to. + * @param event The event we're listening for to receives values from. + * @param options Any extra options. + */ + public constructor( + emitter: EventEmitter, + event: string, + options: EventIteratorOptions = {}, + ) { + this.emitter = emitter; + this.event = event; + this.map = options.map ?? ((args) => args); + this.#limit = options.limit ?? Infinity; + this.#idle = options.idle; + this.filter = options.filter ?? ((): boolean => true); + this.onEnd = options.onEnd; + + // This timer is to idle out on lack of valid responses + if (this.#idle) { + // NOTE: this same code is in next in case when we can't use refresh + this.#idleTimer = setTimeout(this.end.bind(this), this.#idle); + } + this.#push = this.push.bind(this); + const maxListeners = this.emitter.getMaxListeners(); + if (maxListeners !== 0) this.emitter.setMaxListeners(maxListeners + 1); + + this.emitter.on(this.event, this.#push); + } + + /** + * Whether or not the EventIterator has ended. + */ + public get ended(): boolean { + return this.#ended; + } + + /** + * Ends the EventIterator. + */ + public end(): void { + if (this.#ended) return; + this.#ended = true; + this.#queue = []; + + this.emitter.off(this.event, this.#push); + const maxListeners = this.emitter.getMaxListeners(); + if (maxListeners !== 0) { + this.emitter.setMaxListeners(maxListeners - 1); + } + this.onEnd?.(this); + } + // aliases to match usage in NATS and CoCalc. + close = this.end; + stop = this.end; + + drain(): void { + // just immediately end + this.end(); + // [ ] TODO: for compat. I'm not sure what this should be + // or if it matters... + // console.log("WARNING: TODO -- event-iterator drain not implemented"); + } + + /** + * The next value that's received from the EventEmitter. + */ + public async next(): Promise> { + if (this.err) { + const err = this.err; + delete this.err; + this.end(); + throw err; + } + // If there are elements in the queue, return an undone response: + if (this.#queue.length) { + const value = this.#queue.shift()!; + if (!this.filter(value)) { + return this.next(); + } + if (++this.#passed >= this.#limit) { + this.end(); + } + if (this.#idleTimer) { + if (this.#idleTimer.refresh != null) { + this.#idleTimer.refresh(); + } else { + clearTimeout(this.#idleTimer); + this.#idleTimer = setTimeout(this.end.bind(this), this.#idle); + } + } + + return { done: false, value }; + } + + // If the iterator ended, clean-up timer and return a done response: + if (this.#ended) { + if (this.#idleTimer) clearTimeout(this.#idleTimer); + return { done: true, value: undefined as never }; + } + + // Listen for a new element from the emitter: + return new Promise>((resolve) => { + let idleTimer: NodeJS.Timeout | undefined | null = null; + + // If there is an idle time set, we will create a temporary timer, + // which will cause the iterator to end if no new elements are received: + if (this.#idle) { + idleTimer = setTimeout(() => { + this.end(); + resolve(this.next()); + }, this.#idle); + } + + // Once it has received at least one value, we will clear the timer (if defined), + // and resolve with the new value: + this.emitter.once(this.event, () => { + if (idleTimer) clearTimeout(idleTimer); + resolve(this.next()); + }); + }); + } + + /** + * Handles what happens when you break or return from a loop. + */ + public return(): Promise> { + this.end(); + return Promise.resolve({ done: true, value: undefined as never }); + } + + public throw(err): Promise> { + this.err = err; + // fake event to trigger handling of err + this.emitter.emit(this.event); + this.end(); + return Promise.resolve({ done: true, value: undefined as never }); + } + + /** + * The symbol allowing EventIterators to be used in for-await-of loops. + */ + public [Symbol.asyncIterator](): AsyncIterableIterator { + return this; + } + + /** + * Pushes a value into the queue. + */ + protected push(...args): void { + try { + const value = this.map(args); + this.#queue.push(value); + } catch (err) { + this.err = err; + // fake event to trigger handling of err + this.emitter.emit(this.event); + } + } +} diff --git a/src/packages/util/jupyter/api-types.ts b/src/packages/util/jupyter/api-types.ts new file mode 100644 index 0000000000..1a6ae84150 --- /dev/null +++ b/src/packages/util/jupyter/api-types.ts @@ -0,0 +1,21 @@ +export interface ProjectJupyterApiOptions { + hash?: string; // give either hash *or* kernel, input, history, etc. + kernel: string; // jupyter kernel + input: string; // input code to execute + history?: string[]; // optional history of this conversation as a list of input strings. Do not include output. + path?: string; // optional path where execution happens + pool?: { size?: number; timeout_s?: number }; + limits?: Partial<{ + // see packages/jupyter/nbgrader/jupyter-run.ts + timeout_ms_per_cell: number; + max_output_per_cell: number; + max_output: number; + total_output: number; + timeout_ms?: number; + start_time?: number; + }>; +} + +export interface JupyterApiOptions extends ProjectJupyterApiOptions { + project_id: string; +} diff --git a/src/packages/util/jupyter/nbgrader-types.ts b/src/packages/util/jupyter/nbgrader-types.ts index 3b76f4ec1f..36fb49bd9d 100644 --- a/src/packages/util/jupyter/nbgrader-types.ts +++ b/src/packages/util/jupyter/nbgrader-types.ts @@ -95,3 +95,20 @@ export interface JupyterNotebook { nbformat_minor: number; cells: Cell[]; } + +// For tracking limits during the run: +export interface Limits { + timeout_ms_per_cell: number; + max_output_per_cell: number; + max_output: number; + total_output: number; + timeout_ms?: number; + start_time?: number; +} + +export const DEFAULT_LIMITS: Limits = { + timeout_ms_per_cell: 60 * 1000, + max_output_per_cell: 500000, + max_output: 4000000, + total_output: 10000000, +}; diff --git a/src/packages/util/key-value-store.ts b/src/packages/util/key-value-store.ts index 1efe317bfc..b2d886ed73 100644 --- a/src/packages/util/key-value-store.ts +++ b/src/packages/util/key-value-store.ts @@ -42,17 +42,17 @@ class KeyValueStore { // supported by modern browsers value = value.freeze(); // so doesn't get mutated } - if (this._data) this._data[json(key)] = value; + if (this._data) this._data[json(key)!] = value; } get(key) { this.assert_not_closed(); - return this._data?.[json(key)]; + return this._data?.[json(key)!]; } delete(key) { this.assert_not_closed(); - delete this._data?.[json(key)]; + delete this._data?.[json(key)!]; } close() { diff --git a/src/packages/util/message.d.ts b/src/packages/util/message.d.ts index 1edb1e30dd..2682106b9a 100644 --- a/src/packages/util/message.d.ts +++ b/src/packages/util/message.d.ts @@ -14,10 +14,6 @@ export const sign_in_failed: any; export const signed_in: any; export const sign_out: any; export const signed_out: any; -export const send_verification_email: any; -export const change_email_address: any; -export const changed_email_address: any; -export const unlink_passport: any; export const error: any; export const success: any; export const reconnect: any; @@ -34,17 +30,9 @@ export const text_file_read_from_project: any; export const write_file_to_project: any; export const write_text_file_to_project: any; export const file_written_to_project: any; -export const add_license_to_project: any; -export const remove_license_from_project: any; export const project_users: any; -export const invite_collaborator: any; -export const add_collaborator: any; -export const remove_collaborator: any; -export const invite_noncloud_collaborators: any; -export const invite_noncloud_collaborators_resp: any; export const version: any; export const save_blob: any; -export const remove_blob_ttls: any; export const storage: any; export const projects_running_on_server: any; export const local_hub: any; @@ -53,13 +41,11 @@ export const copy_path_between_projects_response: any; export const copy_path_status: any; export const copy_path_status_response: any; export const copy_path_delete: any; -export const project_set_quotas: any; export const print_to_pdf: any; export const printed_to_pdf: any; export const heartbeat: any; export const ping: any; export const pong: any; -export const copy_public_path_between_projects: any; export const log_client_error: any; export const webapp_error: any; export const stripe_get_customer: any; @@ -101,10 +87,7 @@ export const sagews_output_ack: any; export const sagews_interrupt: any; export const sagews_quit: any; export const sagews_start: any; -export const get_syncdoc_history: any; -export const syncdoc_history: any; export const user_tracking: any; -export const admin_reset_password: any; export const purchase_license: any; export const purchase_license_resp: any; export const chatgpt: any; @@ -119,6 +102,4 @@ export const jupyter_execute: any; export const jupyter_execute_response: any; export const jupyter_kernels: any; export const jupyter_start_pool: any; -export const forgot_password: any; -export const reset_forgot_password: any; export const signal_sent: any; diff --git a/src/packages/util/message.js b/src/packages/util/message.js index dcbdc963b4..5aea5143a8 100644 --- a/src/packages/util/message.js +++ b/src/packages/util/message.js @@ -238,7 +238,6 @@ message({ cell_id: undefined, }); // if set, eval scope contains an object cell that refers to the cell in the worksheet with this id. - //########################################### // Account Management //############################################ @@ -425,131 +424,6 @@ message({ id: undefined, }); -message({ - event: "send_verification_email", - id: undefined, - account_id: required, - only_verify: undefined, -}); // usually true, if false the full "welcome" email is sent - -// client --> hub -API( - message2({ - event: "change_email_address", - fields: { - id: { - init: undefined, - desc: "A unique UUID for the query", - }, - account_id: { - init: required, - desc: "account_id for account whose email address is changed", - }, - old_email_address: { - init: "", - desc: "ignored -- deprecated", - }, - new_email_address: { - init: required, - desc: "", - }, - password: { - init: "", - desc: "", - }, - }, - desc: `\ -Given the \`account_id\` for an account, set a new email address. - -Examples: - -Successful change of email address. -\`\`\` - curl -u sk_abcdefQWERTY090900000000: \\ - -d account_id=99ebde5c-58f8-4e29-b6e4-b55b8fd71a1b \\ - -d password=secret_password \\ - -d new_email_address=new@email.com \\ - https://cocalc.com/api/v1/change_email_address - ==> {"event":"changed_email_address", - "id":"8f68f6c4-9851-4b88-bd65-37cb983298e3", - "error":false} -\`\`\` - -Fails if new email address is already in use. - -\`\`\` - curl -u sk_abcdefQWERTY090900000000: \\ - -d account_id=99ebde5c-58f8-4e29-b6e4-b55b8fd71a1b \\ - -d password=secret_password \\ - -d new_email_address=used@email.com \\ - https://cocalc.com/api/v1/change_email_address - ==> {"event":"changed_email_address", - "id":"4501f022-a57c-4aaf-9cd8-af0eb05ebfce", - "error":"email_already_taken"} -\`\`\` - -**Note:** \`account_id\` and \`password\` must match the \`id\` of the current login.\ -`, - }), -); - -// hub --> client -message({ - event: "changed_email_address", - id: undefined, - error: false, // some other error - ttl: undefined, -}); // if user is trying to change password too often, this is time to wait - -// Unlink a passport auth for this account. -// client --> hub -message2({ - event: "unlink_passport", - fields: { - strategy: { - init: required, - desc: "passport strategy", - }, - id: { - init: required, - desc: "numeric id for user and passport strategy", - }, - }, - desc: `\ -Unlink a passport auth for the account. - -Strategies are defined in the database and may be viewed at [/auth/strategies](https://cocalc.com/auth/strategies). - -Example: - -Get passport id for some strategy for current user. -\`\`\` - curl -u sk_abcdefQWERTY090900000000: \\ - -H "Content-Type: application/json" \\ - -d '{"query":{"accounts":{"account_id":"e6993694-820d-4f78-bcc9-10a8e336a88d","passports":null}}}' \\ - https://cocalc.com/api/v1/query - ==> {"query":{"accounts":{"account_id":"e6993694-820d-4f78-bcc9-10a8e336a88d", - "passports":{"facebook-14159265358":{"id":"14159265358",...}}}}, - "multi_response":false, - "event":"query", - "id":"a2554ec8-665b-495b-b0e2-8e248b54eb94"} -\`\`\` - -Unlink passport for that strategy and id. -\`\`\` - curl -u sk_abcdefQWERTY090900000000: \\ - -d strategy=facebook \\ - -d id=14159265358 \\ - https://cocalc.com/api/v1/unlink_passport - ==> {"event":"success", - "id":"14159265358"} -\`\`\` - -Note that success is returned regardless of whether or not passport was linked -for the given strategy and id before issuing the API command.\ -`, -}); - message({ event: "error", id: undefined, @@ -962,64 +836,6 @@ message({ // Managing multiple projects //########################################### -//# search --------------------------- - -API( - message2({ - event: "add_license_to_project", - fields: { - id: { - init: undefined, - desc: "A unique UUID for the message", - }, - project_id: { - init: required, - desc: "project_id", - }, - license_id: { - init: required, - desc: "id of a license", - }, - }, - desc: `\ -Add a license to a project. - -Example: -\`\`\` - example not available yet -\`\`\`\ -`, - }), -); - -API( - message2({ - event: "remove_license_from_project", - fields: { - id: { - init: undefined, - desc: "A unique UUID for the message", - }, - project_id: { - init: required, - desc: "project_id", - }, - license_id: { - init: required, - desc: "id of a license", - }, - }, - desc: `\ -Remove a license from a project. - -Example: -\`\`\` - example not available yet -\`\`\`\ -`, - }), -); - // hub --> client message({ event: "user_search_results", @@ -1034,244 +850,6 @@ message({ users: required, }); // list of {account_id:?, first_name:?, last_name:?, mode:?, state:?} -API( - message2({ - event: "invite_collaborator", - fields: { - id: { - init: undefined, - desc: "A unique UUID for the query", - }, - project_id: { - init: required, - desc: "project_id of project into which user is invited", - }, - account_id: { - init: required, - desc: "account_id of invited user", - }, - title: { - init: undefined, - desc: "Title of the project", - }, - link2proj: { - init: undefined, - desc: "The full URL link to the project", - }, - replyto: { - init: undefined, - desc: "Email address of user who is inviting someone", - }, - replyto_name: { - init: undefined, - desc: "Name of user who is inviting someone", - }, - email: { - init: undefined, - desc: "Body of email user is sending (plain text or HTML)", - }, - subject: { - init: undefined, - desc: "Subject line of invitation email", - }, - }, - desc: `\ -Invite a user who already has a CoCalc account to -become a collaborator on a project. You must be owner -or collaborator on the target project. The user -will receive an email notification. - -Example: -\`\`\` - curl -u sk_abcdefQWERTY090900000000: \\ - -d account_id=99ebde5c-58f8-4e29-b6e4-b55b8fd71a1b \\ - -d project_id=18955da4-4bfa-4afa-910c-7f2358c05eb8 \\ - https://cocalc.com/api/v1/invite_collaborator - ==> {"event":"success", - "id":"e80fd64d-fd7e-4cbc-981c-c0e8c843deec"} -\`\`\`\ -`, - }), -); - -API( - message2({ - event: "add_collaborator", - fields: { - id: { - init: undefined, - desc: "A unique UUID for the query", - }, - project_id: { - init: undefined, - desc: "project_id of project to add user to (can be an array to add multiple users to multiple projects); isn't needed if token_id is specified", - }, - account_id: { - init: required, - desc: "account_id of user (can be an array to add multiple users to multiple projects)", - }, - token_id: { - init: undefined, - desc: "project_invite_token that is needed in case the user **making the request** is not already a project collab", - }, - }, - desc: `\ -Directly add a user to a CoCalc project. -You must be owner or collaborator on the target project or provide, -an optional valid token_id (the token determines the project). -The user is NOT notified via email that they were added, and there -is no confirmation process. (Eventually, there will be -an accept process, or this endpoint will only work -with a notion of "managed accounts".) - -You can optionally add multiple user to multiple projects by padding -an array of strings for project_id and account_id. The arrays -must have the same length. - -Example: -\`\`\` - curl -u sk_abcdefQWERTY090900000000: \\ - -d account_id=99ebde5c-58f8-4e29-b6e4-b55b8fd71a1b \\ - -d project_id=18955da4-4bfa-4afa-910c-7f2358c05eb8 \\ - https://cocalc.com/api/v1/add_collaborator - ==> {"event":"success", - "id":"e80fd64d-fd7e-4cbc-981c-c0e8c843deec"} -\`\`\`\ -`, - }), -); - -API( - message2({ - event: "remove_collaborator", - fields: { - id: { - init: undefined, - desc: "A unique UUID for the query", - }, - project_id: { - init: required, - desc: "project_id of project from which user is removed", - }, - account_id: { - init: required, - desc: "account_id of removed user", - }, - }, - desc: `\ -Remove a user from a CoCalc project. -You must be owner or collaborator on the target project. -The project owner cannot be removed. -The user is NOT notified via email that they were removed. - -Example: -\`\`\` - curl -u sk_abcdefQWERTY090900000000: \\ - -d account_id=99ebde5c-58f8-4e29-b6e4-b55b8fd71a1b \\ - -d project_id=18955da4-4bfa-4afa-910c-7f2358c05eb8 \\ - https://cocalc.com/api/v1/remove_collaborator - ==> {"event":"success", - "id":"e80fd64d-fd7e-4cbc-981c-c0e8c843deec"} -\`\`\`\ -`, - }), -); - -// DANGER -- can be used to spam people. -API( - message2({ - event: "invite_noncloud_collaborators", - fields: { - id: { - init: undefined, - desc: "A unique UUID for the query", - }, - project_id: { - init: required, - desc: "project_id of project into which users are invited", - }, - to: { - init: required, - desc: "comma- or semicolon-delimited string of email addresses", - }, - email: { - init: required, - desc: "body of the email to be sent, may include HTML markup", - }, - title: { - init: required, - desc: "string that will be used for project title in the email", - }, - link2proj: { - init: required, - desc: "URL for the target project", - }, - replyto: { - init: undefined, - desc: "Reply-To email address", - }, - replyto_name: { - init: undefined, - desc: "Reply-To name", - }, - subject: { - init: undefined, - desc: "email Subject", - }, - }, - desc: `\ -Invite users who do not already have a CoCalc account -to join a project. -An invitation email is sent to each user in the \`to\` -option. -Invitation is not sent if there is already a CoCalc -account with the given email address. -You must be owner or collaborator on the target project. - -Limitations: -- Total length of the request message must be less than or equal to 1024 characters. -- Length of each email address must be less than 128 characters. - - -Example: -\`\`\` - curl -u sk_abcdefQWERTY090900000000: \\ - -d project_id=18955da4-4bfa-4afa-910c-7f2358c05eb8 \\ - -d to=someone@m.local \\ - -d 'email=Please sign up and join this project.' \\ - -d 'title=Class Project' \\ - -d link2proj=https://cocalc.com/projects/18955da4-4bfa-4afa-910c-7f2358c05eb8 \\ - https://cocalc.com/api/v1/invite_noncloud_collaborators - ==> {"event":"invite_noncloud_collaborators_resp", - "id":"39d7203d-89b1-4145-8a7a-59e41d5682a3", - "mesg":"Invited someone@m.local to collaborate on a project."} -\`\`\` - -Email sent by the previous example: - -\`\`\` -To: someone@m.local -From: CoCalc
    \\n -To accept the invitation, please sign up at\\n -https://cocalc.com\\n -using exactly the email address 'someone@m.local'.\\n -Then go to -the project 'Team Project'.
    -\`\`\`\ -`, - }), -); - -message({ - event: "invite_noncloud_collaborators_resp", - id: undefined, - mesg: required, -}); - /* Send/receive the current webapp code version number. @@ -1304,23 +882,6 @@ message({ error: undefined, }); // if not saving, a message explaining why. -// remove the ttls from blobs in the blobstore. -// client --> hub -message({ - event: "remove_blob_ttls", - id: undefined, - uuids: required, -}); // list of sha1 hashes of blobs stored in the blobstore - -// DEPRECATED -- used by bup_server -message({ - event: "storage", - action: required, // open, save, snapshot, latest_snapshot, close - project_id: undefined, - param: undefined, - id: undefined, -}); - message({ event: "projects_running_on_server", id: undefined, @@ -1532,22 +1093,6 @@ You need to have read/write access to the associated src/target project. // Admin Functionality //############################################ -// client --> hub; will result in an error if the user is not in the admin group. -message({ - event: "project_set_quotas", - id: undefined, - project_id: required, // the id of the project's id to set. - memory: undefined, // RAM in megabytes - memory_request: undefined, // RAM in megabytes - cpu_shares: undefined, // fair sharing with everybody is 256, not 1 !!! - cores: undefined, // integer max number of cores user can use (>=1) - disk_quota: undefined, // disk quota in megabytes - mintime: undefined, // time in **seconds** until idle projects are terminated - network: undefined, // 1 or 0; if 1, full access to outside network - member_host: undefined, // 1 or 0; if 1, project will be run on a members-only machine - always_running: undefined, // 1 or 0: if 1, project kept running. -}); - /* Printing Files */ @@ -1626,79 +1171,6 @@ message({ now: undefined, }); // timestamp -API( - message2({ - event: "copy_public_path_between_projects", - fields: { - id: { - init: undefined, - desc: "A unique UUID for the query", - }, - src_project_id: { - init: required, - desc: "id of source project", - }, - src_path: { - init: required, - desc: "relative path of directory or file in the source project", - }, - target_project_id: { - init: required, - desc: "id of target project", - }, - target_path: { - init: undefined, - desc: "defaults to src_path", - }, - overwrite_newer: { - init: false, - desc: "overwrite newer versions of file at destination (destructive)", - }, - delete_missing: { - init: false, - desc: "delete files in dest that are missing from source (destructive)", - }, - backup: { - init: false, - desc: "make ~ backup files instead of overwriting changed files", - }, - timeout: { - init: undefined, - desc: "how long to wait for the copy to complete before reporting error (though it could still succeed)", - }, - exclude: { - init: undefined, - desc: "array of rsync patterns to exclude; each item in this string[] array is passed as a --exclude option to rsync", - }, - }, - desc: `\ -Copy a file or directory from a public project to a target project. - -**Note:** the \`timeout\` option is passed to a call to the \`rsync\` command. -If no data is transferred for the specified number of seconds, then -the copy terminates. The default is 0, which means no timeout. - -**Note:** You need to have write access to the target project. - -Example: - -Copy public file \`PUBLIC/doc.txt\` from source project to private file -\`A/sample.txt\` in target project. - -\`\`\` - curl -u sk_abcdefQWERTY090900000000: \\ - -d src_project_id=e49e86aa-192f-410b-8269-4b89fd934fba \\ - -d src_path=PUBLIC/doc.txt \\ - -d target_project_id=2aae4347-214d-4fd1-809c-b327150442d8 \\ - -d target_path=A/sample.txt \\ - https://cocalc.com/api/v1/copy_public_path_between_projects - ==> {"event":"success", - "id":"45d851ac-5ea0-4aea-9997-99a06c054a60"} -\`\`\`\ -`, - }), -); - API( message2({ event: "log_client_error", @@ -2424,23 +1896,6 @@ message({ path: required, }); -// client --> hub -API( - message({ - event: "get_syncdoc_history", - id: undefined, - string_id: required, - patches: undefined, - }), -); - -// hub --> client -message({ - event: "syncdoc_history", - id: undefined, - history: required, -}); - // client --> hub // It's an error if user is not signed in, since // then we don't know who to track. @@ -2451,22 +1906,6 @@ message({ value: required, // map -- additional info about that event }); -// Client <--> hub. -// Enables admins (and only admins!) to generate and get a password reset -// for another user. The response message contains a password reset link, -// though without the site part of the url (the client should fill that in). -// This makes it possible for admins to reset passwords of users, even if -// email is not setup, e.g., for cocalc-docker, and also deals with the -// possibility that users have no email address, or broken email, or they -// can't receive email due to crazy spam filtering. -// Non-admins always get back an error. The reset expires after **8 hours**. -message({ - event: "admin_reset_password", - id: undefined, - email_address: required, - link: undefined, -}); - // Request to purchase a license (either via stripe or a quote) API( message({ diff --git a/src/packages/util/misc.ts b/src/packages/util/misc.ts index f6f96df678..853b5be89e 100644 --- a/src/packages/util/misc.ts +++ b/src/packages/util/misc.ts @@ -1998,6 +1998,20 @@ export function has_null_leaf(obj: object): boolean { return false; } +// mutate obj and delete any undefined leafs. +// was used for MsgPack -- but the ignoreUndefined:true option +// to the encoder is a much better fix. +// export function removeUndefinedLeafs(obj: object) { +// for (const k in obj) { +// const v = obj[k]; +// if (v === undefined) { +// delete obj[k]; +// } else if (is_object(v)) { +// removeUndefinedLeafs(v); +// } +// } +// } + // Peer Grading // This function takes a list of student_ids, // and a number N of the desired number of peers per student. diff --git a/src/packages/util/nats.ts b/src/packages/util/nats.ts deleted file mode 100644 index 5e2ed053e7..0000000000 --- a/src/packages/util/nats.ts +++ /dev/null @@ -1,15 +0,0 @@ -// Some very generic nats related parameters - -// how frequently -export const NATS_OPEN_FILE_TOUCH_INTERVAL = 30000; - -export const CONNECT_OPTIONS = { - // this pingInterval determines how long (worse case) from when the connection dies - // and comes back, until nats starts working again. - pingInterval: 7500, - reconnectTimeWait: 750, - // never give up attempting to reconnect. The default is 10 attempts, but if we allow for - // giving up, then we have to write logic throughout our code to do basically the same - // thing as this, but worse. - maxReconnectAttempts: -1, -} as const; diff --git a/src/packages/util/package.json b/src/packages/util/package.json index f00c4981a9..08c46a2089 100644 --- a/src/packages/util/package.json +++ b/src/packages/util/package.json @@ -16,34 +16,25 @@ }, "scripts": { "preinstall": "npx only-allow pnpm", + "clean": "rm -rf node_modules dist", "build": "pnpm exec tsc --build", "tsc": "pnpm exec tsc --watch --pretty --preserveWatchOutput", "test": "pnpm exec jest", + "depcheck": "pnpx depcheck --ignores events", "prepublishOnly": "pnpm test" }, - "files": [ - "dist/**", - "bin/**", - "README.md", - "package.json" - ], + "files": ["dist/**", "bin/**", "README.md", "package.json"], "author": "SageMath, Inc.", - "keywords": [ - "utilities", - "mathjax", - "markdown", - "cocalc" - ], + "keywords": ["utilities", "mathjax", "markdown", "cocalc"], "license": "SEE LICENSE.md", "dependencies-COMMENT": "We must install react so that react-intl doesn't install the rwrong version! See https://github.com/sagemathinc/cocalc/issues/8132", "dependencies": { "@ant-design/colors": "^6.0.0", "@cocalc/util": "workspace:*", - "@isaacs/ttlcache": "^1.2.1", + "@isaacs/ttlcache": "^1.4.1", "async": "^1.5.2", "awaiting": "^3.0.0", "dayjs": "^1.11.11", - "debug": "^4.4.0", "decimal.js-light": "^2.5.1", "events": "3.3.0", "get-random-values": "^1.2.0", @@ -70,20 +61,13 @@ }, "homepage": "https://github.com/sagemathinc/cocalc/tree/master/src/packages/util", "devDependencies": { - "@types/debug": "^4.1.12", "@types/json-stable-stringify": "^1.0.32", "@types/lodash": "^4.14.202", "@types/node": "^18.16.14", "@types/seedrandom": "^3.0.8", "@types/uuid": "^8.3.1", - "coffee-cache": "^1.0.2", - "coffee-coverage": "^3.0.1", - "coffeescript": "^2.5.1", "expect": "^26.6.2", "seedrandom": "^3.0.5", - "should": "^7.1.1", - "should-sinon": "0.0.3", - "sinon": "^4.5.0", "tsd": "^0.22.0" } } diff --git a/src/packages/util/refcache.ts b/src/packages/util/refcache.ts index 5435d19198..adcc82ed64 100644 --- a/src/packages/util/refcache.ts +++ b/src/packages/util/refcache.ts @@ -1,7 +1,7 @@ /* A reference counting cache. -See example usage in nats/sync. +See example usage in conat/sync. */ import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; @@ -70,7 +70,7 @@ export default function refCache< cache[key] = obj; count[key] = 1; close[key] = obj.close; - obj.close = async () => { + obj.close = () => { count[key] -= 1; if (VERBOSE) { console.log("refCache: close", { name, key, count: count[key] }); @@ -95,6 +95,11 @@ export default function refCache< get.info = () => { return { name, count: { ...count } }; }; + get.one = (): T | undefined => { + for (const key in cache) { + return cache[key]; + } + }; caches[name] = get; return get; @@ -142,13 +147,13 @@ export function refCacheSync< cache[key] = obj; count[key] = 1; close[key] = obj.close; - obj.close = async () => { + obj.close = () => { count[key] -= 1; if (VERBOSE) { console.log("refCacheSync: close", { name, key, count: count[key] }); } if (count[key] <= 0) { - await close[key]?.(); + close[key]?.(); delete cache[key]; delete count[key]; delete close[key]; @@ -166,6 +171,11 @@ export function refCacheSync< get.info = () => { return { name, count: { ...count } }; }; + get.one = (): T | undefined => { + for (const key in cache) { + return cache[key]; + } + }; caches[name] = get; return get; } diff --git a/src/packages/util/schema-validate.js b/src/packages/util/schema-validate.ts similarity index 76% rename from src/packages/util/schema-validate.js rename to src/packages/util/schema-validate.ts index ea4e90530b..6ddd1aa5f9 100644 --- a/src/packages/util/schema-validate.js +++ b/src/packages/util/schema-validate.ts @@ -8,19 +8,15 @@ // If validation fails, this returns an error message; if validation succeeds, // it returns undefined. The input query may be mutated in place. -let validate_client_query; -const misc = require("./misc"); -const schema = require("./schema"); +import * as misc from "./misc"; +import * as schema from "./schema"; -exports.validate_client_query = function validate_client_query( - query, - account_id -) { +export function validate_client_query(query, account_id: string) { let f, is_set_query, k, S; if (misc.is_array(query)) { // it's an array of queries; validate each separately. - for (let q of query) { - const err = validate_client_query(q); + for (const q of query) { + const err = validate_client_query(q, account_id); if (err != null) { return err; } @@ -37,8 +33,8 @@ exports.validate_client_query = function validate_client_query( if (v.length !== 1) { return warn( `must specify exactly one key in the query (the table name), but ${v.length} keys ${JSON.stringify(v)} were specified -- query=${JSON.stringify( - query - )}` + query, + )}`, ); } const table = v[0]; @@ -61,14 +57,14 @@ exports.validate_client_query = function validate_client_query( } if (is_set_query) { - S = user_query.set; + S = user_query?.set; if (S == null) { - return warn(`no user set queries of '${table}' allowed`); + return warn(`no user set queries of table '${table}' allowed`); } } else { - S = user_query.get; + S = user_query?.get; if (S == null) { - return warn(`no user get queries of '${table}' allowed`); + return warn(`no user get queries of table '${table}' allowed`); } } @@ -90,7 +86,12 @@ exports.validate_client_query = function validate_client_query( for (k in S.fields) { f = S.fields[k]; if (typeof f === "function") { - pattern[k] = f(pattern, schema.client_db, account_id); + const val = f(pattern, schema.client_db, account_id); + if (val != null) { + // only if != null -- this shouldn't change get queries to set queries... + // also if val === undefined don't set that (this broke after switch to MsgPack) + pattern[k] = val; + } } } @@ -102,4 +103,4 @@ exports.validate_client_query = function validate_client_query( } } } -}; +} diff --git a/src/packages/util/smc-version.js b/src/packages/util/smc-version.js index c0e259305f..aa11bd675b 100644 --- a/src/packages/util/smc-version.js +++ b/src/packages/util/smc-version.js @@ -1,2 +1,2 @@ /* autogenerated by the update_version script */ -exports.version=1748131702; +exports.version=1750481166; diff --git a/src/packages/util/throttle.ts b/src/packages/util/throttle.ts new file mode 100644 index 0000000000..2fff3d2596 --- /dev/null +++ b/src/packages/util/throttle.ts @@ -0,0 +1,63 @@ +/* +This is a really simple but incredibly useful little class. +See packages/project/conat/terminal.ts for how to use it to make +it so the terminal sends output at a rate of say "24 frames +per second". +*/ +import { EventEmitter } from "events"; + +export class ThrottleString extends EventEmitter { + private buf: string = ""; + private last = Date.now(); + + constructor(private interval: number) { + super(); + } + + write = (data: string) => { + this.buf += data; + const now = Date.now(); + const timeUntilEmit = this.interval - (now - this.last); + if (timeUntilEmit > 0) { + setTimeout(() => this.write(""), timeUntilEmit); + } else { + this.flush(); + } + }; + + flush = () => { + if (this.buf.length > 0) { + this.emit("data", this.buf); + } + this.buf = ""; + this.last = Date.now(); + }; +} + +export class ThrottleAny extends EventEmitter { + private buf: any[] = []; + private last = Date.now(); + + constructor(private interval: number) { + super(); + } + + write = (data: any) => { + this.buf.push(data); + const now = Date.now(); + const timeUntilEmit = this.interval - (now - this.last); + if (timeUntilEmit > 0) { + setTimeout(() => this.write([]), timeUntilEmit); + } else { + this.flush(); + } + }; + + flush = () => { + if (this.buf.length > 0) { + this.emit("data", this.buf); + } + this.buf = []; + this.last = Date.now(); + }; +} diff --git a/src/packages/util/types/execute-code.ts b/src/packages/util/types/execute-code.ts index 452bdccd78..2b6c5a6abc 100644 --- a/src/packages/util/types/execute-code.ts +++ b/src/packages/util/types/execute-code.ts @@ -35,6 +35,7 @@ export interface ExecuteCodeOptions { command: string; args?: string[]; path?: string; // defaults to home directory; where code is executed from. absolute path or path relative to home directory. + cwd?: string; // absolute path where code excuted from (if path not given) timeout?: number; // timeout in **seconds** ulimit_timeout?: boolean; // If set (the default), use ulimit to ensure a cpu timeout -- don't use when launching a daemon! // This has no effect if bash not true. diff --git a/src/packages/util/upgrades/consts.ts b/src/packages/util/upgrades/consts.ts index 9afac6360f..6e6ae8d409 100644 --- a/src/packages/util/upgrades/consts.ts +++ b/src/packages/util/upgrades/consts.ts @@ -9,7 +9,7 @@ import { upgrades } from "@cocalc/util/upgrade-spec"; // RAM export const MAX_RAM_GB = upgrades.max_per_project.memory / 1000; -export const RAM_DEFAULT_GB = 2; // 2gb highly recommended +export const RAM_DEFAULT_GB = 4; // 4gb highly recommended // CPU export const DEFAULT_CPU = 1; @@ -39,7 +39,7 @@ export const REGULAR: Limits = { max: MAX_CPU, }, ram: { - min: 1, + min: 4, dflt: RAM_DEFAULT_GB, max: MAX_RAM_GB, }, diff --git a/src/scripts/ci.sh b/src/scripts/ci.sh index 5cb20e9f6b..229a5ff07e 100755 --- a/src/scripts/ci.sh +++ b/src/scripts/ci.sh @@ -3,14 +3,10 @@ echo >> ci.log echo "`date` -- 📈 Starting local CoCalc CI." >> ci.log echo "`date` -- 🚧 Waiting for changes in upstream..." >> ci.log -echo "You must ALSO run 'pnpm nats-server-ci' and 'pnpm database' in two other terminals." +echo "You must ALSO run 'pnpm database' in another other terminals." echo "Run 'tail -F ci.log' in a terminal to monitor CI status." while true; do - # wait -- best to do this first, since at initial startup we can't immediately kick off build, - # since nats-server needs to build a little to startup, and they will conflict. - sleep 30 - # Fetch the latest commits from upstream git fetch diff --git a/src/scripts/g-tmux.sh b/src/scripts/g-tmux.sh index 179058ff2c..5f052385f4 100755 --- a/src/scripts/g-tmux.sh +++ b/src/scripts/g-tmux.sh @@ -1,27 +1,24 @@ #!/usr/bin/env bash -echo "Spawning tmux windows with: hub, database, nats-server, rspack or memory monitor..." +echo "Spawning tmux windows with: hub, database, rspack or memory monitor..." export PWD=`pwd` tmux new-session -d -s mysession tmux new-window -t mysession:1 tmux new-window -t mysession:2 -tmux new-window -t mysession:3 sleep 2 tmux send-keys -t mysession:0 '$PWD/scripts/g.sh' C-m sleep 2 tmux send-keys -t mysession:1 'pnpm database' C-m -sleep 2 -tmux send-keys -t mysession:2 'pnpm nats-server' C-m if [ -n "$NO_RSPACK_DEV_SERVER" ]; then sleep 2 -tmux send-keys -t mysession:3 'pnpm rspack' C-m +tmux send-keys -t mysession:2 'pnpm rspack' C-m else sleep 2 -tmux send-keys -t mysession:3 '$PWD/scripts/memory_monitor.py' C-m +tmux send-keys -t mysession:2 '$PWD/scripts/memory_monitor.py' C-m fi tmux attach -t mysession:1 diff --git a/src/scripts/g.sh b/src/scripts/g.sh index 621ea2149d..68c84d070c 100755 --- a/src/scripts/g.sh +++ b/src/scripts/g.sh @@ -4,9 +4,8 @@ export LOGS=`pwd`/logs rm -f $LOGS/log unset INIT_CWD unset PGHOST -export DEBUG="cocalc:*" -#export DEBUG_CONSOLE="yes" -unset DEBUG_CONSOLE +export DEBUG="cocalc:*,-cocalc:silly:*" +export DEBUG_CONSOLE="no" # Set this COCALC_DISABLE_NEXT to something nonempty to disable nextjs entirely # which is very helpful when doing development. diff --git a/src/scripts/nats.conf b/src/scripts/nats.conf deleted file mode 100644 index e1dee3b6fb..0000000000 --- a/src/scripts/nats.conf +++ /dev/null @@ -1,33 +0,0 @@ -jetstream: enabled - -jetstream { - store_dir: data/nats/jetstream -} - -websocket { - listen: "localhost:8443" - no_tls: true - jwt_cookie: "%2F3fa218e5-7196-4020-8b30-e2127847cc4f%2Fport%2F5002cocalc_nats_jwt_cookie" -} - -include ../data/nats/trust.conf - -# configuration of the nats based resolver -resolver { - type: full - # Directory in which the account jwt will be stored - dir: 'data/nats/jwt' - # In order to support jwt deletion, set to true - # If the resolver type is full delete will rename the jwt. - # This is to allow manual restoration in case of inadvertent deletion. - # To restore a jwt, remove the added suffix .delete and restart or send a reload signal. - # To free up storage you must manually delete files with the suffix .delete. - allow_delete: false - # Interval at which a nats-server with a nats based account resolver will compare - # it's state with one random nats based account resolver in the cluster and if needed, - # exchange jwt and converge on the same set of jwt. - interval: "2m" - # Timeout for lookup requests in case an account does not exist locally. - timeout: "1.9s" -} - diff --git a/src/workspaces.py b/src/workspaces.py index 3017bb390c..d82eddc9c6 100755 --- a/src/workspaces.py +++ b/src/workspaces.py @@ -109,7 +109,7 @@ def all_packages() -> List[str]: 'packages/sync', 'packages/sync-client', 'packages/sync-fs', - 'packages/nats', + 'packages/conat', 'packages/backend', 'packages/api-client', 'packages/jupyter', @@ -120,6 +120,7 @@ def all_packages() -> List[str]: 'packages/static', # packages/hub assumes this is built (for webpack dev server) 'packages/server', # packages/next assumes this is built 'packages/database', # packages/next also assumes database is built (or at least the coffeescript in it is) + 'packages/file-server', 'packages/next', 'packages/hub', # hub won't build if next isn't already built ]