feat(envs): remove core envs from the manifest and their sources from the workspace

.bitmap

@@ -50,13 +50,6 @@
             "teambit.node/envs/node-babel-mocha@2.0.3": {}
         }
     },
-    "aspect": {
-        "name": "aspect",
-        "scope": "teambit.harmony",
-        "version": "1.0.1087",
-        "mainFile": "index.ts",
-        "rootDir": "scopes/harmony/aspect"
-    },
     "aspect-docs/babel": {
         "name": "aspect-docs/babel",
         "scope": "teambit.compilation",
@@ -726,13 +719,6 @@
             "teambit.node/envs/node-babel-mocha@2.0.3": {}
         }
     },
-    "env": {
-        "name": "env",
-        "scope": "teambit.envs",
-        "version": "1.0.1087",
-        "mainFile": "index.ts",
-        "rootDir": "scopes/envs/env"
-    },
     "envs": {
         "name": "envs",
         "scope": "teambit.envs",
@@ -1101,13 +1087,6 @@
             "teambit.node/envs/node-babel-mocha@2.0.3": {}
         }
     },
-    "mdx": {
-        "name": "mdx",
-        "scope": "teambit.mdx",
-        "version": "1.0.1088",
-        "mainFile": "index.ts",
-        "rootDir": "scopes/mdx/mdx"
-    },
     "merge-lanes": {
         "name": "merge-lanes",
         "scope": "teambit.lanes",
@@ -1541,13 +1520,6 @@
         "mainFile": "index.ts",
         "rootDir": "scopes/component/new-component-helper"
     },
-    "node": {
-        "name": "node",
-        "scope": "teambit.harmony",
-        "version": "1.0.1087",
-        "mainFile": "index.ts",
-        "rootDir": "scopes/harmony/node"
-    },
     "notifications": {
         "name": "notifications",
         "scope": "teambit.ui-foundation",
@@ -1680,27 +1652,13 @@
         "mainFile": "index.ts",
         "rootDir": "scopes/harmony/pubsub"
     },
-    "react": {
-        "name": "react",
-        "scope": "teambit.react",
-        "version": "1.0.1087",
-        "mainFile": "index.ts",
-        "rootDir": "scopes/react/react"
-    },
     "react-router": {
         "name": "react-router",
         "scope": "teambit.ui-foundation",
         "version": "1.0.1087",
         "mainFile": "index.ts",
         "rootDir": "scopes/ui-foundation/react-router/react-router"
     },
-    "readme": {
-        "name": "readme",
-        "scope": "teambit.mdx",
-        "version": "1.0.1088",
-        "mainFile": "index.ts",
-        "rootDir": "scopes/mdx/readme"
-    },
     "refactoring": {
         "name": "refactoring",
         "scope": "teambit.component",
@@ -2589,4 +2547,4 @@
         "rootDir": "scopes/dependencies/yarn"
     },
     "$schema-version": "17.0.0"
-}
+}

components/legacy/e2e-helper/e2e-env-helper.ts

@@ -1,4 +1,5 @@
 import * as path from 'path';
+import fs from 'fs-extra';
 import { getRootComponentDir } from '@teambit/workspace.root-components';
 import type CommandHelper from './e2e-command-helper';
 import type ExtensionsHelper from './e2e-extensions-helper';
@@ -14,6 +15,52 @@ type SetCustomEnvOpts = {
   skipLink?: boolean;
 };
 
+/**
+ * env packages that the old-format env fixtures import. these envs used to be core aspects, so
+ * their packages were always available. now they need to be installed. the versions should match
+ * the pinned versions in legacy-core-envs.ts (scopes/envs/envs).
+ */
+const FIXTURE_ENV_BASE_PACKAGES: Record<string, string> = {
+  '@teambit/node': '@teambit/node@1.0.1042',
+  '@teambit/react': '@teambit/react@1.0.1042',
+  '@teambit/mdx': '@teambit/mdx@1.0.1043',
+  // not env bases - the tiny runtime deps of the minimal fixture envs (node-env-1/node-env-2),
+  // listed here so setCustomEnv installs them and the fixtures load without MissingPackages
+  'lodash.get': 'lodash.get@4.4.2',
+  'lodash.flatten': 'lodash.flatten@4.4.0',
+};
+
+/**
+ * the env configured on the old-format env fixtures (see setCustomEnv). it used to be a core
+ * aspect, now its package must be installed for the fixture env to be compiled and loaded.
+ */
+const ENVS_ENV_PACKAGE = '@teambit/env@1.0.1042';
+
+/**
+ * peers required by the published legacy env packages (via @teambit/cloud UI hooks and
+ * graphql-request) that yarn's hoisted linker doesn't auto-install (pnpm auto-installs peers).
+ * without them the env fails to load under yarn with "Cannot find module '@apollo/client'" (or
+ * 'graphql').
+ */
+export const NODE_ENV_HOISTED_PEERS = '@apollo/client@3.14.1 graphql@14.7.0';
+
+/**
+ * a modern env.jsonc-based node env, pinned so upstream releases don't change test behavior.
+ * see setBitdevNodeEnv.
+ */
+const BITDEV_NODE_ENV = 'bitdev.node/node-env';
+const BITDEV_NODE_ENV_VERSION = '6.0.16';
+const BITDEV_NODE_ENV_PACKAGE = `@bitdev/node.node-env@${BITDEV_NODE_ENV_VERSION}`;
+export const BITDEV_NODE_ENV_ID = `${BITDEV_NODE_ENV}@${BITDEV_NODE_ENV_VERSION}`;
+
+/**
+ * the env configured on old-format aspect-style env fixtures (see setBabelWithTsHarmony). it used
+ * to be a core aspect, now its package must be installed for the fixture env to be loaded.
+ * the node env is a runtime dependency of the aspect env and must be installed at the root as
+ * well for the aspect env to load.
+ */
+const ASPECT_ENV_PACKAGES = ['@teambit/aspect@1.0.1042', '@teambit/node@1.0.1042'];
+
 export const ENV_POLICY = {
   peers: [
     {
@@ -139,11 +186,115 @@ export default class EnvHelper {
         },
       },
     });
-    this.command.install();
+    this.command.install([...ASPECT_ENV_PACKAGES, ...this.getFixtureEnvBasePackages(EXTENSIONS_BASE_FOLDER)].join(' '));
     this.command.compile();
     return EXTENSIONS_BASE_FOLDER;
   }
 
+  /**
+   * create a minimal env component with zero dependencies - like the core empty-env, just under
+   * a different name. the env plugin file is plain .js, so it needs no compilation (the env
+   * component itself stays on the default env) and no package installation at all.
+   * use it for suites that just need "an env" (or two envs, to switch between) and don't
+   * exercise any actual env functionality (compiler/tester/etc).
+   */
+  setSimpleEnv(name = 'simple-env'): string {
+    this.fs.outputFile(
+      `${name}/${name}.bit-env.js`,
+      `class SimpleEnv {
+  name = '${name}';
+}
+module.exports.default = new SimpleEnv();
+`
+    );
+    // the env manifest is what marks the component as an env before it was ever loaded
+    this.fs.outputFile(`${name}/env.jsonc`, `{}`);
+    this.fs.outputFile(`${name}/index.js`, `module.exports = require('./${name}.bit-env');\n`);
+    this.command.addComponent(name);
+    this.command.link();
+    return name;
+  }
+
+  /**
+   * create a minimal env component that provides only a typescript compiler (workspace compile
+   * and build pipeline), taken from the core teambit.typescript/typescript aspect at runtime -
+   * so like setSimpleEnv, it needs NO package installation and no compilation of the env itself
+   * (the plugin file is plain .js). by default it is configured on all components via variants
+   * (like setBitdevNodeEnv), pass variantPattern=undefined to only create the env component.
+   * use it for suites that need components to actually compile (dists) but don't exercise any
+   * other env functionality (tester/linter/etc). it is much faster than installing an env like
+   * the legacy teambit.harmony/node chain or even the modern node env.
+   */
+  setTsEnv(variantPattern: string | undefined = '*', name = 'ts-env'): string {
+    this.fs.outputFile(
+      `${name}/${name}.bit-env.js`,
+      `class TsEnv {
+  name = '${name}';
+  createTsCompiler(context) {
+    // the env context exposes core aspects. the typescript aspect is a core aspect, so no
+    // installation is needed.
+    const typescript = context.getAspect('teambit.typescript/typescript');
+    return typescript.createCjsCompiler({
+      tsconfig: {
+        compilerOptions: {
+          lib: ['es2019'],
+          target: 'es2019',
+          module: 'commonjs',
+          declaration: true,
+          sourceMap: true,
+          allowJs: true,
+        },
+      },
+      types: [],
+      // compile all capsules in one ts program instead of project-references. it is faster, and
+      // more importantly, project-references require every dependency capsule to have a
+      // tsconfig.json, which the capsule of this env itself (compiler-less, on the default env)
+      // does not have.
+      singleProgramCompilation: true,
+    });
+  }
+  compiler() {
+    return (context) => this.createTsCompiler(context);
+  }
+  package() {
+    // the pkg service only duck-types the PackageGenerator class. without it, the package.json
+    // main of the components points to the source file instead of the dist
+    return () => ({
+      packageJsonProps: { main: 'dist/{main}.js', types: '{main}.ts' },
+      npmIgnore: [],
+    });
+  }
+  build() {
+    // the builder service only duck-types the Pipeline class (an object with compute() returning
+    // the build tasks), so there is no need to import @teambit/builder for Pipeline.from()
+    return {
+      compute: (context) => {
+        const compilerAspect = context.getAspect('teambit.compilation/compiler');
+        return [compilerAspect.createTask('TSCompiler', this.createTsCompiler(context))];
+      },
+    };
+  }
+}
+module.exports.default = new TsEnv();
+`
+    );
+    // the env manifest marks the component as an env before it was ever loaded
+    this.fs.outputFile(`${name}/env.jsonc`, `{}`);
+    this.fs.outputFile(`${name}/index.js`, `module.exports = require('./${name}.bit-env');\n`);
+    this.command.addComponent(name);
+    if (variantPattern) {
+      // the env component gets the workspace default-scope, which is not necessarily the remote
+      // scope (e.g. when the workspace was set up with addRemoteScopeAsDefaultScope: false)
+      const envId = `${this.extensions.workspaceJsonc.getDefaultScope()}/${name}`;
+      this.extensions.addExtensionToVariant(variantPattern, envId, {});
+      this.extensions.addExtensionToVariant(variantPattern, 'teambit.envs/envs', { env: envId });
+      // the env component itself must not get the variant config (an env can't be its own env)
+      this.extensions.workspaceJsonc.addToVariant(name, 'propagate', false);
+    }
+    this.command.link();
+    return name;
+  }
+
   setEmptyEnv() {
     this.fs.outputFile(
       'empty-env/empty-env.bit-env.ts',
@@ -154,18 +305,79 @@ export default new EmptyEnv();
     this.fs.outputFile('empty-env/index.ts', `export { EmptyEnv } from './empty-env.bit-env';`);
     this.command.addComponent('empty-env');
     this.command.setEnv('empty-env', 'teambit.envs/env');
+    // teambit.envs/env is no longer a core aspect - install it so the .bit-env plugin compiles
+    this.command.install(ENVS_ENV_PACKAGE);
+  }
+
+  /**
+   * set the legacy `teambit.harmony/node` env on the given variant and install its package.
+   * this env used to be a core aspect so no installation was needed, now it must be installed
+   * (with the version pinned in legacy-core-envs.ts, see FIXTURE_ENV_BASE_PACKAGES).
+   * `extraPackages` is for peers the env package needs but the package manager doesn't
+   * auto-install (see NODE_ENV_HOISTED_PEERS for yarn).
+   */
+  setNodeEnv(variantPattern = '*', extraPackages = '') {
+    this.extensions.addExtensionToVariant(variantPattern, 'teambit.harmony/node', {});
+    this.command.install(`${FIXTURE_ENV_BASE_PACKAGES['@teambit/node']} ${extraPackages}`.trim());
+    // the env is loaded only at the end of the first install. run a second install so its
+    // dependency policies (e.g. @types/jest as a dev dep) are applied to the components - the
+    // standard flow for old-style envs (see the "run bit install again" suggestion).
+    this.command.install();
+  }
+
+  /**
+   * install the packages of the legacy `teambit.harmony/aspect` env (and its node env runtime
+   * dependency). needed after configuring the aspect env on a component the old way (versionless).
+   */
+  installAspectEnv() {
+    this.command.install(ASPECT_ENV_PACKAGES.join(' '));
+  }
+
+  /**
+   * set the modern `bitdev.node/node-env` env (env.jsonc based) on the given variant and install
+   * its package. much lighter and faster to install than the legacy `teambit.harmony/node` chain
+   * (a single install, no scope-aspect capsules) - prefer it for suites that just need an env
+   * with a typescript compiler and don't test the legacy-env surface itself.
+   * caveats: its build pipeline (tag/snap with --build) compiles with moduleResolution nodenext,
+   * so relative imports in fixture files must include the `.js` extension; its tester is vitest
+   * (not jest) and its linter is oxlint (not eslint).
+   */
+  setBitdevNodeEnv(variantPattern = '*') {
+    this.extensions.addExtensionToVariant(variantPattern, `${BITDEV_NODE_ENV}@${BITDEV_NODE_ENV_VERSION}`, {});
+    this.extensions.addExtensionToVariant(variantPattern, 'teambit.envs/envs', {
+      env: BITDEV_NODE_ENV,
+    });
+    this.command.install(BITDEV_NODE_ENV_PACKAGE);
   }
 
   setCustomEnv(extensionsBaseFolder = 'node-env', options: SetCustomEnvOpts = {}): string {
     this.fixtures.copyFixtureExtensions(extensionsBaseFolder);
     this.command.addComponent(extensionsBaseFolder);
     this.extensions.addExtensionToVariant(extensionsBaseFolder, 'teambit.envs/env');
     if (!options.skipLink) this.command.link();
-    if (!options.skipInstall) this.command.install();
+    if (!options.skipInstall) {
+      this.command.install([ENVS_ENV_PACKAGE, ...this.getFixtureEnvBasePackages(extensionsBaseFolder)].join(' '));
+    }
     if (!options.skipCompile) this.command.compile();
     return extensionsBaseFolder;
   }
 
+  /**
+   * find which env packages the given fixture imports, so they can be installed (see
+   * FIXTURE_ENV_BASE_PACKAGES).
+   */
+  getFixtureEnvBasePackages(extensionsBaseFolder: string): string[] {
+    const extensionDir = path.join(this.scopes.localPath, extensionsBaseFolder);
+    const allContent = fs
+      .readdirSync(extensionDir)
+      .filter((file) => file.endsWith('.ts'))
+      .map((file) => fs.readFileSync(path.join(extensionDir, file)).toString())
+      .join('\n');
+    return Object.keys(FIXTURE_ENV_BASE_PACKAGES)
+      .filter((pkg) => allContent.includes(`'${pkg}'`) || allContent.includes(`"${pkg}"`))
+      .map((pkg) => FIXTURE_ENV_BASE_PACKAGES[pkg]);
+  }
+
   /**
    * This will generate env in the new format (using the *.bit-env.* plugin)
    * @param extensionsBaseFolder
@@ -177,7 +389,12 @@ export default new EmptyEnv();
     envJsoncOptions: GenerateEnvJsoncOptions = { policy: ENV_POLICY },
     runInstall = true,
     targetFolder?: string,
-    id?: string
+    id?: string,
+    // the env of the generated env itself. defaults to the legacy teambit.envs/env. pass the
+    // real-world envs-of-envs (e.g. `bitdev.general/envs/bit-env@x.y.z`, what `bit create
+    // react-env` produces) to exercise the regular-external-env loading path - remember to
+    // include its package in basePackages so it gets installed.
+    envsEnv = 'teambit.envs/env'
   ): string {
     const addOptions = id ? { i: id } : {};
     // Pin the base react-env to a React 18 version, but only for envs that actually pull in react-env.
@@ -203,8 +420,8 @@ export default new EmptyEnv();
     this.fixtures.copyFixtureExtensions(extensionsBaseFolder, undefined, targetFolder);
     this.command.addComponent(targetFolder || extensionsBaseFolder, addOptions);
     this.fixtures.generateEnvJsoncFile(targetFolder || extensionsBaseFolder, envJsoncOptions);
-    this.extensions.addExtensionToVariant(targetFolder || extensionsBaseFolder, 'teambit.envs/env');
-    this.command.setEnv(id || extensionsBaseFolder, 'teambit.envs/env');
+    this.extensions.addExtensionToVariant(targetFolder || extensionsBaseFolder, envsEnv);
+    this.command.setEnv(id || extensionsBaseFolder, envsEnv);
     this.command.link();
     if (runInstall) {
       this.command.install(basePackages.join(' '));

components/legacy/e2e-helper/excluded-fixtures/extensions/babel-env/babel-env.extension.ts

@@ -1,3 +1,7 @@
+// @ts-nocheck
+// the e2e runs inside the bit repo, where core-aspect types (e.g. @teambit/compiler) resolve
+// to the repo sources while the env tree in the capsule brings the published @teambit/builder.
+// the two BuildContext types are structurally identical but nominally different - skip checking.
 import { EnvsMain, EnvsAspect } from '@teambit/envs';
 import { ReactAspect, ReactMain } from '@teambit/react';
 import { BabelAspect, BabelMain } from '@teambit/babel';

components/legacy/e2e-helper/excluded-fixtures/extensions/custom-react-env/custom-react-env.main.runtime.ts

@@ -1,3 +1,7 @@
+// @ts-nocheck
+// the e2e runs inside the bit repo, where core-aspect types (e.g. @teambit/react) resolve to the
+// repo sources while the env tree in the capsule brings the published packages. the types are
+// structurally identical but nominally different - skip checking.
 import { MainRuntime } from '@teambit/cli';
 import { ReactAspect, ReactMain } from '@teambit/react';
 import { EnvsAspect, EnvsMain } from '@teambit/envs';

components/legacy/e2e-helper/excluded-fixtures/extensions/dev-files-env/dev-files-env.extension.ts

@@ -1,3 +1,4 @@
+// @bit-no-check
 // @ts-nocheck
 // the e2e runs inside the bit repo, where core-aspect types (e.g. @teambit/envs) resolve to the
 // repo sources while the env tree in the capsule brings the published packages. the types are

components/legacy/e2e-helper/excluded-fixtures/extensions/env-add-dependencies/add-deps-env.extension.ts

@@ -1,3 +1,4 @@
+// @bit-no-check
 // @ts-nocheck
 // the e2e runs inside the bit repo, where core-aspect types (e.g. @teambit/envs) resolve to the
 // repo sources while the env tree in the capsule brings the published packages. the types are