feat(csv-import-export): demo end-to-end checks, docs, and first release

.changeset/csv-import-export-initial.md

@@ -0,0 +1,5 @@
+---
+"@fbritoferreira/strapi-csv-import-export": minor
+---
+
+First release of the CSV import/export plugin for Strapi 5: import CSV files into any collection type with column mapping, upsert on `documentId` or a unique field, and relation linking by any field of the related collection; export collection types as CSV with chosen columns; and a history of every import and export.

demo/README.md

@@ -4,6 +4,7 @@ A Strapi 5 app linked to this repo's packages, for trying changes against a
 real Strapi instead of mocks:
 
 - `@fbritoferreira/strapi-admin-api` plugin (`/admin-api/*`)
+- `@fbritoferreira/strapi-csv-import-export` plugin (admin menu **CSV Import / Export**)
 - `@fbritoferreira/strapi-provider-email-cloudflare` (only when Cloudflare credentials are set)
 - `@fbritoferreira/strapi` client (`pnpm try-client`, `pnpm generate`)
 - `@strapi/plugin-graphql` (`/graphql`)
@@ -29,8 +30,9 @@ pnpm develop
 ```
 
 The admin panel is at http://localhost:1337/admin. On first boot the app seeds
-two published articles, one draft and the homepage, and lets the public role
-read them.
+two published articles, one draft, the homepage, two categories (`news`,
+`sport`) and three tags (`strapi`, `typescript`, `csv`), and lets the public
+role read the articles and the homepage.
 
 Create an admin from the CLI instead of the browser:
 
@@ -45,7 +47,9 @@ and the email provider) and restart `pnpm develop`.
 
 | UID | Kind | Notes |
 | --- | --- | --- |
-| `api::article.article` | collection | Draft & Publish on: `title`, `slug`, `body` |
+| `api::article.article` | collection | Draft & Publish on: `title`, `slug`, `body`, `views`, `featured`, `category` (manyToOne), `tags` (manyToMany), `cover` (media) |
+| `api::category.category` | collection | `name`, `slug`; no Draft & Publish |
+| `api::tag.tag` | collection | `label` (unique); no Draft & Publish |
 | `api::homepage.homepage` | single | `title`, `description` |
 
 ## Try the admin API plugin
@@ -59,6 +63,21 @@ curl -s -X POST localhost:1337/admin-api/tokens -H "Authorization: Bearer $TOKEN
   -H 'content-type: application/json' -d '{"name":"demo","type":"full-access"}'
 ```
 
+## Try the CSV import/export plugin
+
+With the app running and the admin created:
+
+```bash
+pnpm try-csv
+```
+
+It imports `scripts/articles.csv` twice through the plugin's admin API and
+fails unless the second run creates nothing. It also checks that a missing
+relation target aborts its batch when `onMissingRelation` is `fail`, and that an
+export re-imports on `documentId` as updates only. In the admin panel the same
+flow is under **CSV Import / Export**, and the Article list view has **Import
+CSV** and **Export CSV** buttons.
+
 ## Try the client
 
 ```bash

demo/config/middlewares.ts

@@ -7,5 +7,6 @@ export default [
   'strapi::query',
   'strapi::body',
   'strapi::session',
+  'strapi::favicon',
   'strapi::public',
 ];

demo/config/plugins.ts

@@ -2,6 +2,9 @@ export default ({ env }) => ({
   // @fbritoferreira/strapi-admin-api, linked from packages/admin-api
   'admin-api': { enabled: true },
 
+  // @fbritoferreira/strapi-csv-import-export, linked from packages/csv-import-export
+  'csv-import-export': { enabled: true },
+
   graphql: {
     config: {
       landingPage: true,

demo/package.json

@@ -9,11 +9,13 @@
     "strapi": "strapi",
     "build:admin": "strapi build",
     "generate": "strapi-client generate --dir . -o types/strapi.ts",
-    "try-client": "node scripts/try-client.mjs"
+    "try-client": "node scripts/try-client.mjs",
+    "try-csv": "node scripts/try-csv.mjs"
   },
   "dependencies": {
     "@fbritoferreira/strapi": "link:../packages/client",
     "@fbritoferreira/strapi-admin-api": "link:../packages/admin-api",
+    "@fbritoferreira/strapi-csv-import-export": "link:../packages/csv-import-export",
     "@fbritoferreira/strapi-provider-email-cloudflare": "link:../packages/provider-email-cloudflare",
     "@strapi/plugin-graphql": "5.57.0",
     "@strapi/plugin-users-permissions": "5.57.0",

demo/scripts/articles.csv

@@ -0,0 +1,6 @@
+slug,title,views,featured,category,tags
+hello-strapi,Hello Strapi (imported),10,true,news,strapi
+csv-import,CSV import,5,false,sport,csv|typescript
+csv-export,CSV export,,yes,news,
+bad-views,Bad views,many,false,news,
+missing-category,Missing category,1,false,cooking,

demo/scripts/try-csv.mjs

@@ -0,0 +1,209 @@
+// Exercise the CSV import/export plugin against the running demo app through
+// its admin API: import scripts/articles.csv twice and check the second run
+// updates instead of creating, then export and read the job history.
+import { strict as assert } from 'node:assert';
+import { readFile } from 'node:fs/promises';
+
+const base = process.env.STRAPI_URL ?? 'http://localhost:1337';
+const email = process.env.ADMIN_EMAIL ?? 'admin@demo.local';
+const password = process.env.ADMIN_PASSWORD ?? 'DemoPass123';
+const ARTICLE = 'api::article.article';
+
+const login = await fetch(`${base}/admin/login`, {
+  method: 'POST',
+  headers: { 'content-type': 'application/json' },
+  body: JSON.stringify({ email, password }),
+});
+assert.equal(login.status, 200, `admin login failed; create the admin first (see README)`);
+const { token } = (await login.json()).data;
+
+const request = async (bearer, method, path, body) => {
+  const res = await fetch(`${base}${path}`, {
+    method,
+    headers: { authorization: `Bearer ${bearer}`, 'content-type': 'application/json' },
+    body: body && JSON.stringify(body),
+  });
+  const text = await res.text();
+  return { status: res.status, body: text ? JSON.parse(text) : null };
+};
+
+const api = async (method, path, body) => {
+  const res = await request(token, method, `/csv-import-export${path}`, body);
+  assert.ok(res.status < 300, `${method} ${path} -> ${res.status} ${JSON.stringify(res.body)}`);
+  return res.body;
+};
+
+/** Real imports are logged, so each one needs a running job. */
+const importWithJob = async (config, rows) => {
+  const { data: job } = await api('POST', '/jobs', {
+    uid: ARTICLE,
+    status: config.status,
+    fileName: 'try-csv',
+    totalRows: rows.length,
+    config,
+  });
+  const { data: result } = await api('POST', `/import/${ARTICLE}`, { ...config, dryRun: false, jobId: job.id, rowOffset: 0, rows });
+  await api('POST', `/jobs/${job.id}/finish`, { state: result.aborted ? 'failed' : 'completed' });
+  return result;
+};
+
+// The fixture has no quoted fields, so a plain split is enough here.
+const [header, ...lines] = (await readFile(new URL('./articles.csv', import.meta.url), 'utf8')).trim().split('\n');
+const columns = header.split(',');
+const rows = lines.map((line) => Object.fromEntries(line.split(',').map((cell, i) => [columns[i], cell])));
+
+const runImport = async () => {
+  const config = {
+    status: 'published',
+    matchField: 'slug',
+    mapping: { slug: 'slug', title: 'title', views: 'views', featured: 'featured', category: 'category', tags: 'tags' },
+    relations: { category: { matchOn: 'slug' }, tags: { matchOn: 'label' } },
+    onMissingRelation: 'skip',
+  };
+  const { data: job } = await api('POST', '/jobs', {
+    uid: ARTICLE,
+    status: config.status,
+    fileName: 'articles.csv',
+    totalRows: rows.length,
+    config,
+  });
+  const { data: result } = await api('POST', `/import/${ARTICLE}`, { ...config, dryRun: false, jobId: job.id, rowOffset: 0, rows });
+  await api('POST', `/jobs/${job.id}/finish`, { state: 'completed' });
+  const { data: logged } = await api('GET', `/jobs/${job.id}`);
+  const count = (action) => result.results.filter((r) => r.action === action).length;
+  return { result, logged, created: count('created'), updated: count('updated'), skipped: count('skipped'), errored: count('error') };
+};
+
+const first = await runImport();
+console.log('first import', { created: first.created, updated: first.updated, skipped: first.skipped, errored: first.errored });
+for (const r of first.result.results.filter((r) => r.error)) console.log(`  row ${r.row}: ${r.action}: ${r.error}`);
+
+const second = await runImport();
+console.log('second import', { created: second.created, updated: second.updated, skipped: second.skipped, errored: second.errored });
+
+assert.equal(second.created, 0, 'second import must not create anything');
+assert.equal(second.updated, first.created + first.updated, 'second import must update every row the first one wrote');
+assert.equal(second.skipped, 1, 'missing-category row is skipped');
+assert.equal(second.errored, 1, 'bad-views row is an error');
+assert.equal(second.logged.state, 'completed');
+assert.equal(second.logged.updated, second.updated, 'job log counts match the batch result');
+assert.equal(second.logged.errors.length, 2, 'job log keeps the skipped and failed rows only');
+
+const { data: exported } = await api('POST', `/export/${ARTICLE}`, {
+  status: 'published',
+  columns: [
+    { field: 'slug', header: 'slug' },
+    { field: 'title', header: 'title' },
+    { field: 'category', header: 'category', matchOn: 'slug' },
+    { field: 'tags', header: 'tags', matchOn: 'label' },
+  ],
+});
+const { csv } = exported;
+assert.match(csv, /^slug,title,category,tags\r\n/);
+// Relations landed on the published version, read back through the export.
+assert.match(csv, /csv-import,CSV import,sport,(csv\|typescript|typescript\|csv)\r\n/);
+assert.match(csv, /hello-strapi,Hello Strapi \(imported\),news,strapi\r\n/);
+assert.doesNotMatch(csv, /missing-category|bad-views/, 'skipped and failed rows were never written');
+console.log(`export: ${csv.trim().split('\r\n').length - 1} rows`);
+
+// A row whose relation target is missing aborts its whole batch with onMissingRelation: fail.
+const failed = await importWithJob(
+  {
+    status: 'draft',
+    matchField: 'slug',
+    mapping: { slug: 'slug', category: 'category' },
+    relations: { category: { matchOn: 'slug' } },
+    onMissingRelation: 'fail',
+  },
+  [
+    { slug: 'should-not-exist', category: 'news' },
+    { slug: 'missing-category', category: 'cooking' },
+  ]
+);
+assert.equal(failed.aborted, true, 'fail mode aborts the batch');
+const check = await api('POST', `/export/${ARTICLE}`, { status: 'draft', columns: [{ field: 'slug', header: 'slug' }] });
+assert.doesNotMatch(check.data.csv, /should-not-exist/, 'nothing in an aborted batch is written');
+console.log('fail mode: batch aborted, nothing written');
+
+// Round trip: the export re-imports on documentId with no creates.
+const roundTrip = await api('POST', `/export/${ARTICLE}`, {
+  status: 'published',
+  columns: [
+    { field: 'documentId', header: 'documentId' },
+    { field: 'views', header: 'views' },
+  ],
+});
+const [, ...exportedLines] = roundTrip.data.csv.trim().split('\r\n');
+const reimport = await importWithJob(
+  {
+    status: 'published',
+    matchField: 'documentId',
+    mapping: { documentId: 'documentId', views: 'views' },
+    relations: {},
+    onMissingRelation: 'skip',
+  },
+  exportedLines.map((line) => {
+    const [documentId, views] = line.split(',');
+    return { documentId, views };
+  })
+);
+assert.ok(reimport.results.every((r) => r.action === 'updated'), 'every exported row updates its own entry');
+console.log(`round trip on documentId: ${reimport.results.length} updated, 0 created`);
+
+// Field-level permissions: an editor who may read only title and slug cannot export views.
+const EDITOR = { email: 'csv-editor@demo.local', password: 'EditorPass123' };
+const roles = (await request(token, 'GET', '/admin/roles')).body.data;
+let role = roles.find((r) => r.name === 'CSV editor');
+if (!role) {
+  role = (await request(token, 'POST', '/admin/roles', { name: 'CSV editor', description: 'try-csv' })).body.data;
+}
+await request(token, 'PUT', `/admin/roles/${role.id}/permissions`, {
+  permissions: [
+    {
+      action: 'plugin::content-manager.explorer.read',
+      subject: ARTICLE,
+      properties: { fields: ['title', 'slug'] },
+      conditions: [],
+    },
+    { action: 'plugin::csv-import-export.export', subject: null, properties: {}, conditions: [] },
+  ],
+});
+let editorLogin = await request('', 'POST', '/admin/login', EDITOR);
+if (editorLogin.status !== 200) {
+  const invited = await request(token, 'POST', '/admin/users', {
+    email: EDITOR.email,
+    firstname: 'CSV',
+    lastname: 'Editor',
+    roles: [role.id],
+  });
+  await request('', 'POST', '/admin/register', {
+    registrationToken: invited.body.data.registrationToken,
+    userInfo: { firstname: 'CSV', lastname: 'Editor', password: EDITOR.password },
+  });
+  editorLogin = await request('', 'POST', '/admin/login', EDITOR);
+}
+const editorToken = editorLogin.body.data.token;
+const exportAs = (columns) =>
+  request(editorToken, 'POST', `/csv-import-export/export/${ARTICLE}`, { status: 'published', columns });
+const allowed = await exportAs([{ field: 'title', header: 'title' }]);
+assert.equal(allowed.status, 200, `editor exports title: ${JSON.stringify(allowed.body)}`);
+const refused = await exportAs([
+  { field: 'title', header: 'title' },
+  { field: 'views', header: 'views' },
+]);
+assert.equal(refused.status, 403, 'editor cannot export a field outside the role');
+const importRefused = await request(editorToken, 'POST', '/csv-import-export/jobs', {
+  uid: ARTICLE,
+  status: 'draft',
+  totalRows: 1,
+  config: {},
+});
+assert.equal(importRefused.status, 403, 'editor without the import permission cannot start an import');
+console.log(`permissions: editor export of title ${allowed.status}, of views ${refused.status}, import ${importRefused.status}`);
+
+const { data: history, meta } = await api('GET', `/jobs?uid=${ARTICLE}`);
+assert.ok(history.length >= 3, 'history lists both imports and the export');
+assert.ok(!('errors' in history[0]), 'history list omits the error rows');
+console.log(`history: ${meta.pagination.total} jobs, latest ${history[0].kind} ${history[0].state}`);
+
+console.log('csv import/export ok');

demo/src/api/article/content-types/article/schema.json

@@ -20,6 +20,29 @@
     },
     "body": {
       "type": "text"
+    },
+    "views": {
+      "type": "integer"
+    },
+    "featured": {
+      "type": "boolean",
+      "default": false
+    },
+    "category": {
+      "type": "relation",
+      "relation": "manyToOne",
+      "target": "api::category.category",
+      "inversedBy": "articles"
+    },
+    "tags": {
+      "type": "relation",
+      "relation": "manyToMany",
+      "target": "api::tag.tag"
+    },
+    "cover": {
+      "type": "media",
+      "multiple": false,
+      "allowedTypes": ["images"]
     }
   }
 }

demo/src/api/category/content-types/category/schema.json

@@ -0,0 +1,28 @@
+{
+  "kind": "collectionType",
+  "collectionName": "categories",
+  "info": {
+    "singularName": "category",
+    "pluralName": "categories",
+    "displayName": "Category"
+  },
+  "options": {
+    "draftAndPublish": false
+  },
+  "attributes": {
+    "name": {
+      "type": "string",
+      "required": true
+    },
+    "slug": {
+      "type": "uid",
+      "targetField": "name"
+    },
+    "articles": {
+      "type": "relation",
+      "relation": "oneToMany",
+      "target": "api::article.article",
+      "mappedBy": "category"
+    }
+  }
+}

demo/src/api/category/controllers/category.ts

@@ -0,0 +1,3 @@
+import { factories } from '@strapi/strapi';
+
+export default factories.createCoreController('api::category.category');

demo/src/api/category/routes/category.ts

@@ -0,0 +1,3 @@
+import { factories } from '@strapi/strapi';
+
+export default factories.createCoreRouter('api::category.category');

demo/src/api/category/services/category.ts

@@ -0,0 +1,3 @@
+import { factories } from '@strapi/strapi';
+
+export default factories.createCoreService('api::category.category');