Make bulk registration CSV import more user-friendly (#411)
* Refactor CSV import * Finish refactoring * Add types file * update readme
This commit is contained in:
Borislav Pantaleev
GitHub
parent
28043dc2ca
commit
7af2831be8
@ -110,6 +110,7 @@ The following changes are already implemented:
|
|||||||
* 🗑️ [Add "Purge Remote Media" button](https://github.com/etkecc/synapse-admin/pull/237)
|
* 🗑️ [Add "Purge Remote Media" button](https://github.com/etkecc/synapse-admin/pull/237)
|
||||||
* [Respect base url (`BASE_PATH` / `vite build --base`) when loading `config.json`](https://github.com/etkecc/synapse-admin/pull/274)
|
* [Respect base url (`BASE_PATH` / `vite build --base`) when loading `config.json`](https://github.com/etkecc/synapse-admin/pull/274)
|
||||||
* [Add Users' Account Data tab](https://github.com/etkecc/synapse-admin/pull/276)
|
* [Add Users' Account Data tab](https://github.com/etkecc/synapse-admin/pull/276)
|
||||||
|
* [Make bulk registration CSV import more user-friendly](https://github.com/etkecc/synapse-admin/pull/411)
|
||||||
|
|
||||||
#### exclusive for [etke.cc](https://etke.cc) customers
|
#### exclusive for [etke.cc](https://etke.cc) customers
|
||||||
|
|
||||||
|
|||||||
@ -6,7 +6,7 @@ import { Admin, CustomRoutes, Resource, resolveBrowserLocale } from "react-admin
|
|||||||
import { Route } from "react-router-dom";
|
import { Route } from "react-router-dom";
|
||||||
|
|
||||||
import AdminLayout from "./components/AdminLayout";
|
import AdminLayout from "./components/AdminLayout";
|
||||||
import UserImport from "./components/UserImport";
|
import UserImport from "./components/user-import/UserImport";
|
||||||
import germanMessages from "./i18n/de";
|
import germanMessages from "./i18n/de";
|
||||||
import englishMessages from "./i18n/en";
|
import englishMessages from "./i18n/en";
|
||||||
import frenchMessages from "./i18n/fr";
|
import frenchMessages from "./i18n/fr";
|
||||||
|
|||||||
@ -1,575 +0,0 @@
|
|||||||
import { parse as parseCsv, unparse as unparseCsv, ParseResult } from "papaparse";
|
|
||||||
import { ChangeEvent, useState } from "react";
|
|
||||||
|
|
||||||
import {
|
|
||||||
Button,
|
|
||||||
Card,
|
|
||||||
CardActions,
|
|
||||||
CardContent,
|
|
||||||
CardHeader,
|
|
||||||
Checkbox,
|
|
||||||
Container,
|
|
||||||
FormControlLabel,
|
|
||||||
NativeSelect,
|
|
||||||
} from "@mui/material";
|
|
||||||
import { DataProvider, useTranslate } from "ra-core";
|
|
||||||
import { useDataProvider, useNotify, RaRecord, Title } from "react-admin";
|
|
||||||
|
|
||||||
import { generateRandomMXID, returnMXID } from "../utils/mxid";
|
|
||||||
import { generateRandomPassword } from "../utils/password";
|
|
||||||
|
|
||||||
const LOGGING = true;
|
|
||||||
|
|
||||||
const expectedFields = ["id", "displayname"].sort();
|
|
||||||
|
|
||||||
function TranslatableOption({ value, text }) {
|
|
||||||
const translate = useTranslate();
|
|
||||||
return <option value={value}>{translate(text)}</option>;
|
|
||||||
}
|
|
||||||
|
|
||||||
type Progress = {
|
|
||||||
done: number;
|
|
||||||
limit: number;
|
|
||||||
} | null;
|
|
||||||
|
|
||||||
interface ImportLine {
|
|
||||||
id: string;
|
|
||||||
displayname: string;
|
|
||||||
user_type?: string;
|
|
||||||
name?: string;
|
|
||||||
deactivated?: boolean;
|
|
||||||
guest?: boolean;
|
|
||||||
admin?: boolean;
|
|
||||||
is_admin?: boolean;
|
|
||||||
password?: string;
|
|
||||||
avatar_url?: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
interface ChangeStats {
|
|
||||||
total: number;
|
|
||||||
id: number;
|
|
||||||
is_guest: number;
|
|
||||||
admin: number;
|
|
||||||
password: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
interface ImportResult {
|
|
||||||
skippedRecords: RaRecord[];
|
|
||||||
erroredRecords: RaRecord[];
|
|
||||||
succeededRecords: RaRecord[];
|
|
||||||
totalRecordCount: number;
|
|
||||||
changeStats: ChangeStats;
|
|
||||||
wasDryRun: boolean;
|
|
||||||
}
|
|
||||||
|
|
||||||
const FilePicker = () => {
|
|
||||||
const [values, setValues] = useState<ImportLine[]>([]);
|
|
||||||
const [error, setError] = useState<string | string[] | null>(null);
|
|
||||||
const [stats, setStats] = useState<ChangeStats | null>(null);
|
|
||||||
const [dryRun, setDryRun] = useState(true);
|
|
||||||
|
|
||||||
const [progress, setProgress] = useState<Progress>(null);
|
|
||||||
|
|
||||||
const [importResults, setImportResults] = useState<ImportResult | null>(null);
|
|
||||||
const [skippedRecords, setSkippedRecords] = useState<string>("");
|
|
||||||
|
|
||||||
const [conflictMode, setConflictMode] = useState("stop");
|
|
||||||
const [passwordMode, setPasswordMode] = useState(true);
|
|
||||||
const [useridMode, setUseridMode] = useState("update");
|
|
||||||
|
|
||||||
const translate = useTranslate();
|
|
||||||
const notify = useNotify();
|
|
||||||
|
|
||||||
const dataProvider = useDataProvider();
|
|
||||||
|
|
||||||
const onFileChange = async (e: ChangeEvent<HTMLInputElement>) => {
|
|
||||||
if (progress !== null) return;
|
|
||||||
|
|
||||||
setValues([]);
|
|
||||||
setError(null);
|
|
||||||
setStats(null);
|
|
||||||
setImportResults(null);
|
|
||||||
const file = e.target.files ? e.target.files[0] : null;
|
|
||||||
if (!file) return;
|
|
||||||
/* Let's refuse some unreasonably big files instead of freezing
|
|
||||||
* up the browser */
|
|
||||||
if (file.size > 100000000) {
|
|
||||||
const message = translate("import_users.errors.unreasonably_big", {
|
|
||||||
size: (file.size / (1024 * 1024)).toFixed(2),
|
|
||||||
});
|
|
||||||
notify(message);
|
|
||||||
setError(message);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
parseCsv<ImportLine>(file, {
|
|
||||||
header: true,
|
|
||||||
skipEmptyLines: true /* especially for a final EOL in the csv file */,
|
|
||||||
complete: result => {
|
|
||||||
if (result.errors) {
|
|
||||||
setError(result.errors.map(e => e.toString()));
|
|
||||||
}
|
|
||||||
/* Papaparse is very lenient, we may be able to salvage
|
|
||||||
* the data in the file. */
|
|
||||||
verifyCsv(result, { setValues, setStats, setError });
|
|
||||||
},
|
|
||||||
});
|
|
||||||
} catch {
|
|
||||||
setError("Unknown error");
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
const verifyCsv = ({ data, meta, errors }: ParseResult<ImportLine>, { setValues, setStats, setError }) => {
|
|
||||||
/* First, verify the presence of required fields */
|
|
||||||
meta.fields = meta.fields?.map(f => f.trim().toLowerCase());
|
|
||||||
const missingFields = expectedFields.filter(eF => !meta.fields?.find(mF => eF === mF));
|
|
||||||
|
|
||||||
if (missingFields.length > 0) {
|
|
||||||
setError(translate("import_users.error.required_field", { field: missingFields[0] }));
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
// XXX after deciding on how "name" and friends should be handled below,
|
|
||||||
// this place will want changes, too.
|
|
||||||
|
|
||||||
/* Collect some stats to prevent sneaky csv files from adding admin
|
|
||||||
users or something.
|
|
||||||
*/
|
|
||||||
const stats = {
|
|
||||||
user_types: { default: 0 },
|
|
||||||
is_guest: 0,
|
|
||||||
admin: 0,
|
|
||||||
deactivated: 0,
|
|
||||||
password: 0,
|
|
||||||
avatar_url: 0,
|
|
||||||
id: 0,
|
|
||||||
|
|
||||||
total: data.length,
|
|
||||||
};
|
|
||||||
|
|
||||||
const errorMessages = errors.map(e => e.message);
|
|
||||||
// sanitize the data first
|
|
||||||
data = data.map(line => {
|
|
||||||
const newLine = {} as ImportLine;
|
|
||||||
for (const [key, value] of Object.entries(line)) {
|
|
||||||
newLine[key.trim().toLowerCase()] = value;
|
|
||||||
}
|
|
||||||
return newLine;
|
|
||||||
});
|
|
||||||
// process the data
|
|
||||||
data.forEach((line, idx) => {
|
|
||||||
if (line.user_type === undefined || line.user_type === "") {
|
|
||||||
stats.user_types.default++;
|
|
||||||
} else {
|
|
||||||
stats.user_types[line.user_type] += 1;
|
|
||||||
}
|
|
||||||
/* XXX correct the csv export that react-admin offers for the users
|
|
||||||
* resource so it gives sensible field names and doesn't duplicate
|
|
||||||
* id as "name"?
|
|
||||||
*/
|
|
||||||
if (meta.fields?.includes("name")) {
|
|
||||||
delete line.name;
|
|
||||||
}
|
|
||||||
if (meta.fields?.includes("user_type")) {
|
|
||||||
delete line.user_type;
|
|
||||||
}
|
|
||||||
if (meta.fields?.includes("is_admin")) {
|
|
||||||
delete line.is_admin;
|
|
||||||
}
|
|
||||||
|
|
||||||
["is_guest", "admin", "deactivated"].forEach(f => {
|
|
||||||
if (line[f] === "true") {
|
|
||||||
stats[f]++;
|
|
||||||
line[f] = true; // we need true booleans instead of strings
|
|
||||||
} else {
|
|
||||||
if (line[f] !== "false" && line[f] !== "") {
|
|
||||||
console.log("invalid value", line[f], "for field " + f + " in row " + idx);
|
|
||||||
errorMessages.push(
|
|
||||||
translate("import_users.error.invalid_value", {
|
|
||||||
field: f,
|
|
||||||
row: idx,
|
|
||||||
})
|
|
||||||
);
|
|
||||||
}
|
|
||||||
line[f] = false; // default values to false
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
if (line.password !== undefined && line.password !== "") {
|
|
||||||
stats.password++;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (line.avatar_url !== undefined && line.avatar_url !== "") {
|
|
||||||
stats.avatar_url++;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (line.id !== undefined && line.id !== "") {
|
|
||||||
stats.id++;
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
if (errorMessages.length > 0) {
|
|
||||||
setError(errorMessages);
|
|
||||||
}
|
|
||||||
setStats(stats);
|
|
||||||
setValues(data);
|
|
||||||
|
|
||||||
return true;
|
|
||||||
};
|
|
||||||
|
|
||||||
const runImport = async () => {
|
|
||||||
if (progress !== null) {
|
|
||||||
notify("import_users.errors.already_in_progress");
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const results = await doImport(
|
|
||||||
dataProvider,
|
|
||||||
values,
|
|
||||||
conflictMode,
|
|
||||||
passwordMode,
|
|
||||||
useridMode,
|
|
||||||
dryRun,
|
|
||||||
setProgress,
|
|
||||||
setError
|
|
||||||
);
|
|
||||||
setImportResults(results);
|
|
||||||
// offer CSV download of skipped or errored records
|
|
||||||
// (so that the user doesn't have to filter out successful
|
|
||||||
// records manually when fixing stuff in the CSV)
|
|
||||||
setSkippedRecords(unparseCsv(results.skippedRecords));
|
|
||||||
if (LOGGING) console.log("Skipped records:");
|
|
||||||
if (LOGGING) console.log(skippedRecords);
|
|
||||||
};
|
|
||||||
|
|
||||||
// XXX every single one of the requests will restart the activity indicator
|
|
||||||
// which doesn't look very good.
|
|
||||||
|
|
||||||
const doImport = async (
|
|
||||||
dataProvider: DataProvider,
|
|
||||||
data: ImportLine[],
|
|
||||||
conflictMode: string,
|
|
||||||
passwordMode: boolean,
|
|
||||||
useridMode: string,
|
|
||||||
dryRun: boolean,
|
|
||||||
setProgress: (progress: Progress) => void,
|
|
||||||
setError: (message: string) => void
|
|
||||||
): Promise<ImportResult> => {
|
|
||||||
const skippedRecords: ImportLine[] = [];
|
|
||||||
const erroredRecords: ImportLine[] = [];
|
|
||||||
const succeededRecords: ImportLine[] = [];
|
|
||||||
const changeStats: ChangeStats = {
|
|
||||||
total: 0,
|
|
||||||
id: 0,
|
|
||||||
is_guest: 0,
|
|
||||||
admin: 0,
|
|
||||||
password: 0,
|
|
||||||
};
|
|
||||||
let entriesDone = 0;
|
|
||||||
const entriesCount = data.length;
|
|
||||||
try {
|
|
||||||
setProgress({ done: entriesDone, limit: entriesCount });
|
|
||||||
for (const entry of data) {
|
|
||||||
const userRecord = { ...entry };
|
|
||||||
// No need to do a bunch of cryptographic random number getting if
|
|
||||||
// we are using neither a generated password nor a generated user id.
|
|
||||||
if (useridMode === "ignore" || userRecord.id === undefined || userRecord.id === "") {
|
|
||||||
userRecord.id = generateRandomMXID();
|
|
||||||
}
|
|
||||||
if (passwordMode === false || entry.password === undefined || entry.password === "") {
|
|
||||||
userRecord.password = generateRandomPassword();
|
|
||||||
}
|
|
||||||
// we want to ensure that the ID is always full MXID, otherwise randomly-generated MXIDs will be in the full
|
|
||||||
// form, but the ones from the CSV will be localpart-only.
|
|
||||||
userRecord.id = returnMXID(userRecord.id);
|
|
||||||
/* TODO record update stats (especially admin no -> yes, deactivated x -> !x, ... */
|
|
||||||
|
|
||||||
/* For these modes we will consider the ID that's in the record.
|
|
||||||
* If the mode is "stop", we will not continue adding more records, and
|
|
||||||
* we will offer information on what was already added and what was
|
|
||||||
* skipped.
|
|
||||||
*
|
|
||||||
* If the mode is "skip", we record the record for later, but don't
|
|
||||||
* send it to the server.
|
|
||||||
*
|
|
||||||
* If the mode is "update", we change fields that are reasonable to
|
|
||||||
* update.
|
|
||||||
* - If the "password mode" is "true" (i.e. "use passwords from csv"):
|
|
||||||
* - if the record has a password
|
|
||||||
* - send the password along with the record
|
|
||||||
* - if the record has no password
|
|
||||||
* - generate a new password
|
|
||||||
* - If the "password mode" is "false"
|
|
||||||
* - never generate a new password to update existing users with
|
|
||||||
*/
|
|
||||||
|
|
||||||
/* We just act as if there are no IDs in the CSV, so every user will be
|
|
||||||
* created anew.
|
|
||||||
* We do a simple retry loop so that an accidental hit on an existing ID
|
|
||||||
* doesn't trip us up.
|
|
||||||
*/
|
|
||||||
if (LOGGING) console.log("will check for existence of record " + JSON.stringify(userRecord));
|
|
||||||
let retries = 0;
|
|
||||||
const submitRecord = (recordData: ImportLine) => {
|
|
||||||
return dataProvider.getOne("users", { id: recordData.id }).then(
|
|
||||||
async () => {
|
|
||||||
if (LOGGING) console.log("already existed");
|
|
||||||
|
|
||||||
if (useridMode === "update" || conflictMode === "skip") {
|
|
||||||
skippedRecords.push(recordData);
|
|
||||||
} else if (conflictMode === "stop") {
|
|
||||||
throw new Error(
|
|
||||||
translate("import_users.error.id_exits", {
|
|
||||||
id: recordData.id,
|
|
||||||
})
|
|
||||||
);
|
|
||||||
} else {
|
|
||||||
const newRecordData = Object.assign({}, recordData, {
|
|
||||||
id: generateRandomMXID(),
|
|
||||||
});
|
|
||||||
retries++;
|
|
||||||
if (retries > 512) {
|
|
||||||
console.warn("retry loop got stuck? pathological situation?");
|
|
||||||
skippedRecords.push(recordData);
|
|
||||||
} else {
|
|
||||||
await submitRecord(newRecordData);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
async () => {
|
|
||||||
if (LOGGING) console.log("OK to create record " + recordData.id + " (" + recordData.displayname + ").");
|
|
||||||
|
|
||||||
if (!dryRun) {
|
|
||||||
await dataProvider.create("users", { data: recordData });
|
|
||||||
}
|
|
||||||
succeededRecords.push(recordData);
|
|
||||||
}
|
|
||||||
);
|
|
||||||
};
|
|
||||||
|
|
||||||
await submitRecord(userRecord);
|
|
||||||
entriesDone++;
|
|
||||||
setProgress({ done: entriesDone, limit: data.length });
|
|
||||||
}
|
|
||||||
|
|
||||||
setProgress(null);
|
|
||||||
} catch (e) {
|
|
||||||
setError(
|
|
||||||
translate("import_users.error.at_entry", {
|
|
||||||
entry: entriesDone + 1,
|
|
||||||
message: e instanceof Error ? e.message : String(e),
|
|
||||||
})
|
|
||||||
);
|
|
||||||
setProgress(null);
|
|
||||||
}
|
|
||||||
return {
|
|
||||||
skippedRecords,
|
|
||||||
erroredRecords,
|
|
||||||
succeededRecords,
|
|
||||||
totalRecordCount: entriesCount,
|
|
||||||
changeStats,
|
|
||||||
wasDryRun: dryRun,
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
const downloadSkippedRecords = () => {
|
|
||||||
const element = document.createElement("a");
|
|
||||||
console.log(skippedRecords);
|
|
||||||
const file = new Blob([skippedRecords], {
|
|
||||||
type: "text/comma-separated-values",
|
|
||||||
});
|
|
||||||
element.href = URL.createObjectURL(file);
|
|
||||||
element.download = "skippedRecords.csv";
|
|
||||||
document.body.appendChild(element); // Required for this to work in FireFox
|
|
||||||
element.click();
|
|
||||||
};
|
|
||||||
|
|
||||||
const onConflictModeChanged = async (e: ChangeEvent<HTMLSelectElement>) => {
|
|
||||||
if (progress !== null) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const value = e.target.value;
|
|
||||||
setConflictMode(value);
|
|
||||||
};
|
|
||||||
|
|
||||||
const onPasswordModeChange = (e: ChangeEvent<HTMLInputElement>) => {
|
|
||||||
if (progress !== null) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
setPasswordMode(e.target.checked);
|
|
||||||
};
|
|
||||||
|
|
||||||
const onUseridModeChanged = async (e: ChangeEvent<HTMLSelectElement>) => {
|
|
||||||
if (progress !== null) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const value = e.target.value;
|
|
||||||
setUseridMode(value);
|
|
||||||
};
|
|
||||||
|
|
||||||
const onDryRunModeChanged = (e: ChangeEvent<HTMLInputElement>) => {
|
|
||||||
if (progress !== null) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
setDryRun(e.target.checked);
|
|
||||||
};
|
|
||||||
|
|
||||||
// render individual small components
|
|
||||||
|
|
||||||
const statsCards = stats &&
|
|
||||||
!importResults && [
|
|
||||||
<Container>
|
|
||||||
<CardHeader title={translate("import_users.cards.importstats.header")} />
|
|
||||||
<CardContent>
|
|
||||||
<div>{translate("import_users.cards.importstats.users_total", stats.total)}</div>
|
|
||||||
<div>{translate("import_users.cards.importstats.guest_count", stats.is_guest)}</div>
|
|
||||||
<div>{translate("import_users.cards.importstats.admin_count", stats.admin)}</div>
|
|
||||||
</CardContent>
|
|
||||||
</Container>,
|
|
||||||
<Container>
|
|
||||||
<CardHeader title={translate("import_users.cards.ids.header")} />
|
|
||||||
<CardContent>
|
|
||||||
<div>
|
|
||||||
{stats.id === stats.total
|
|
||||||
? translate("import_users.cards.ids.all_ids_present")
|
|
||||||
: translate("import_users.cards.ids.count_ids_present", stats.id)}
|
|
||||||
</div>
|
|
||||||
{stats.id > 0 ? (
|
|
||||||
<div>
|
|
||||||
<NativeSelect onChange={onUseridModeChanged} value={useridMode} disabled={progress !== null}>
|
|
||||||
<TranslatableOption value="ignore" text="import_users.cards.ids.mode.ignore" />
|
|
||||||
<TranslatableOption value="update" text="import_users.cards.ids.mode.update" />
|
|
||||||
</NativeSelect>
|
|
||||||
</div>
|
|
||||||
) : (
|
|
||||||
""
|
|
||||||
)}
|
|
||||||
</CardContent>
|
|
||||||
</Container>,
|
|
||||||
<Container>
|
|
||||||
<CardHeader title={translate("import_users.cards.passwords.header")} />
|
|
||||||
<CardContent>
|
|
||||||
<div>
|
|
||||||
{stats.password === stats.total
|
|
||||||
? translate("import_users.cards.passwords.all_passwords_present")
|
|
||||||
: translate("import_users.cards.passwords.count_passwords_present", stats.password)}
|
|
||||||
</div>
|
|
||||||
{stats.password > 0 ? (
|
|
||||||
<div>
|
|
||||||
<FormControlLabel
|
|
||||||
control={
|
|
||||||
<Checkbox checked={passwordMode} disabled={progress !== null} onChange={onPasswordModeChange} />
|
|
||||||
}
|
|
||||||
label={translate("import_users.cards.passwords.use_passwords")}
|
|
||||||
/>
|
|
||||||
</div>
|
|
||||||
) : (
|
|
||||||
""
|
|
||||||
)}
|
|
||||||
</CardContent>
|
|
||||||
</Container>,
|
|
||||||
];
|
|
||||||
|
|
||||||
const conflictCards = stats && !importResults && (
|
|
||||||
<Container>
|
|
||||||
<CardHeader title={translate("import_users.cards.conflicts.header")} />
|
|
||||||
<CardContent>
|
|
||||||
<div>
|
|
||||||
<NativeSelect onChange={onConflictModeChanged} value={conflictMode} disabled={progress !== null}>
|
|
||||||
<TranslatableOption value="stop" text="import_users.cards.conflicts.mode.stop" />
|
|
||||||
<TranslatableOption value="skip" text="import_users.cards.conflicts.mode.skip" />
|
|
||||||
</NativeSelect>
|
|
||||||
</div>
|
|
||||||
</CardContent>
|
|
||||||
</Container>
|
|
||||||
);
|
|
||||||
|
|
||||||
const errorCards = error && (
|
|
||||||
<Container>
|
|
||||||
<CardHeader title={translate("import_users.error.error")} />
|
|
||||||
<CardContent>
|
|
||||||
{(Array.isArray(error) ? error : [error]).map(e => (
|
|
||||||
<div>{e}</div>
|
|
||||||
))}
|
|
||||||
</CardContent>
|
|
||||||
</Container>
|
|
||||||
);
|
|
||||||
|
|
||||||
const uploadCard = !importResults && (
|
|
||||||
<Container>
|
|
||||||
<CardHeader title={translate("import_users.cards.upload.header")} />
|
|
||||||
<CardContent>
|
|
||||||
{translate("import_users.cards.upload.explanation")}
|
|
||||||
<a href="./data/example.csv">example.csv</a>
|
|
||||||
<br />
|
|
||||||
<br />
|
|
||||||
<input type="file" onChange={onFileChange} disabled={progress !== null} />
|
|
||||||
</CardContent>
|
|
||||||
</Container>
|
|
||||||
);
|
|
||||||
|
|
||||||
const resultsCard = importResults && (
|
|
||||||
<CardContent>
|
|
||||||
<CardHeader title={translate("import_users.cards.results.header")} />
|
|
||||||
<div>
|
|
||||||
{translate("import_users.cards.results.total", importResults.totalRecordCount)}
|
|
||||||
<br />
|
|
||||||
{translate("import_users.cards.results.successful", importResults.succeededRecords.length)}
|
|
||||||
<br />
|
|
||||||
{importResults.skippedRecords.length
|
|
||||||
? [
|
|
||||||
translate("import_users.cards.results.skipped", importResults.skippedRecords.length),
|
|
||||||
<div>
|
|
||||||
<button onClick={downloadSkippedRecords}>
|
|
||||||
{translate("import_users.cards.results.download_skipped")}
|
|
||||||
</button>
|
|
||||||
</div>,
|
|
||||||
<br />,
|
|
||||||
]
|
|
||||||
: ""}
|
|
||||||
{importResults.erroredRecords.length
|
|
||||||
? [translate("import_users.cards.results.skipped", importResults.erroredRecords.length), <br />]
|
|
||||||
: ""}
|
|
||||||
<br />
|
|
||||||
{importResults.wasDryRun && [translate("import_users.cards.results.simulated_only"), <br />]}
|
|
||||||
</div>
|
|
||||||
</CardContent>
|
|
||||||
);
|
|
||||||
|
|
||||||
const startImportCard =
|
|
||||||
!values || values.length === 0 || importResults ? undefined : (
|
|
||||||
<CardActions>
|
|
||||||
<FormControlLabel
|
|
||||||
control={<Checkbox checked={dryRun} onChange={onDryRunModeChanged} disabled={progress !== null} />}
|
|
||||||
label={translate("import_users.cards.startImport.simulate_only")}
|
|
||||||
/>
|
|
||||||
<Button size="large" onClick={runImport} disabled={progress !== null}>
|
|
||||||
{translate("import_users.cards.startImport.run_import")}
|
|
||||||
</Button>
|
|
||||||
{progress !== null ? (
|
|
||||||
<div>
|
|
||||||
{progress.done} of {progress.limit} done
|
|
||||||
</div>
|
|
||||||
) : null}
|
|
||||||
</CardActions>
|
|
||||||
);
|
|
||||||
|
|
||||||
const allCards: React.JSX.Element[] = [];
|
|
||||||
if (uploadCard) allCards.push(uploadCard);
|
|
||||||
if (errorCards) allCards.push(errorCards);
|
|
||||||
if (conflictCards) allCards.push(conflictCards);
|
|
||||||
if (statsCards) allCards.push(...statsCards);
|
|
||||||
if (startImportCard) allCards.push(startImportCard);
|
|
||||||
if (resultsCard) allCards.push(resultsCard);
|
|
||||||
|
|
||||||
const cardContainer = <Card>{allCards}</Card>;
|
|
||||||
|
|
||||||
return [<Title defaultTitle={translate("import_users.title")} />, cardContainer];
|
|
||||||
};
|
|
||||||
|
|
||||||
export const UserImport = FilePicker;
|
|
||||||
export default UserImport;
|
|
||||||
40
src/components/user-import/ConflictModeCard.tsx
Normal file
40
src/components/user-import/ConflictModeCard.tsx
Normal file
@ -0,0 +1,40 @@
|
|||||||
|
import { NativeSelect, Paper } from "@mui/material";
|
||||||
|
|
||||||
|
import { CardContent, CardHeader, Container } from "@mui/material";
|
||||||
|
|
||||||
|
import { useTranslate } from "ra-core";
|
||||||
|
import { ParsedStats, Progress } from "./types";
|
||||||
|
import { ChangeEventHandler } from "react";
|
||||||
|
|
||||||
|
const TranslatableOption = ({ value, text }: { value: string, text: string }) => {
|
||||||
|
const translate = useTranslate();
|
||||||
|
return <option value={value}>{translate(text)}</option>;
|
||||||
|
}
|
||||||
|
|
||||||
|
const ConflictModeCard = ({ stats, importResults, onConflictModeChanged, conflictMode, progress }:
|
||||||
|
{ stats: ParsedStats | null, importResults: any, onConflictModeChanged: ChangeEventHandler<HTMLSelectElement>, conflictMode: string, progress: Progress }) => {
|
||||||
|
const translate = useTranslate();
|
||||||
|
|
||||||
|
if (!stats || importResults) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return (
|
||||||
|
<Container sx={{ mb: 3 }}>
|
||||||
|
<Paper elevation={1}>
|
||||||
|
<CardHeader
|
||||||
|
title={translate("import_users.cards.conflicts.header")}
|
||||||
|
sx={{ borderBottom: 1, borderColor: "divider" }}
|
||||||
|
/>
|
||||||
|
<CardContent>
|
||||||
|
<NativeSelect onChange={onConflictModeChanged} value={conflictMode} disabled={progress !== null}>
|
||||||
|
<TranslatableOption value="stop" text="import_users.cards.conflicts.mode.stop" />
|
||||||
|
<TranslatableOption value="skip" text="import_users.cards.conflicts.mode.skip" />
|
||||||
|
</NativeSelect>
|
||||||
|
</CardContent>
|
||||||
|
</Paper>
|
||||||
|
</Container>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export default ConflictModeCard;
|
||||||
43
src/components/user-import/ErrorsCard.tsx
Normal file
43
src/components/user-import/ErrorsCard.tsx
Normal file
@ -0,0 +1,43 @@
|
|||||||
|
import {
|
||||||
|
Container,
|
||||||
|
Paper,
|
||||||
|
CardHeader,
|
||||||
|
CardContent,
|
||||||
|
Stack,
|
||||||
|
Typography,
|
||||||
|
} from "@mui/material";
|
||||||
|
import { useTranslate } from "ra-core";
|
||||||
|
|
||||||
|
const ErrorsCard = ({ errors }: { errors: string[] }) => {
|
||||||
|
const translate = useTranslate();
|
||||||
|
|
||||||
|
if (errors.length === 0) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return (
|
||||||
|
<Container sx={{ mb: 3 }}>
|
||||||
|
<Paper elevation={1}>
|
||||||
|
<CardHeader
|
||||||
|
title={translate("import_users.error.error")}
|
||||||
|
sx={{
|
||||||
|
borderBottom: 1,
|
||||||
|
borderColor: "error.main",
|
||||||
|
color: "error.main"
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
<CardContent>
|
||||||
|
<Stack spacing={1}>
|
||||||
|
{errors.map((e, idx) => (
|
||||||
|
<Typography key={idx} color="error">
|
||||||
|
{e}
|
||||||
|
</Typography>
|
||||||
|
))}
|
||||||
|
</Stack>
|
||||||
|
</CardContent>
|
||||||
|
</Paper>
|
||||||
|
</Container>
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
|
export default ErrorsCard;
|
||||||
75
src/components/user-import/ResultsCard.tsx
Normal file
75
src/components/user-import/ResultsCard.tsx
Normal file
@ -0,0 +1,75 @@
|
|||||||
|
import { Alert, Box, CardContent, CardHeader, Container, List, ListItem, ListItemText, Paper, Stack, Typography } from "@mui/material"
|
||||||
|
import { Button, Link, useTranslate } from "react-admin";
|
||||||
|
import { ImportResult } from "./types";
|
||||||
|
import DownloadIcon from "@mui/icons-material/Download";
|
||||||
|
import ArrowBackIcon from "@mui/icons-material/ArrowBack";
|
||||||
|
|
||||||
|
const ResultsCard = ({ importResults, downloadSkippedRecords }: { importResults: ImportResult | null, downloadSkippedRecords: () => void }) => {
|
||||||
|
const translate = useTranslate();
|
||||||
|
|
||||||
|
if (!importResults) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return (
|
||||||
|
<Container>
|
||||||
|
<Paper>
|
||||||
|
<CardHeader
|
||||||
|
title={translate("import_users.cards.results.header")}
|
||||||
|
pb={0}
|
||||||
|
sx={{
|
||||||
|
borderBottom: 1,
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
<CardContent>
|
||||||
|
<Stack spacing={2}>
|
||||||
|
<Typography key="total" color="text.primary">
|
||||||
|
{translate("import_users.cards.results.total", importResults.totalRecordCount)}
|
||||||
|
</Typography>
|
||||||
|
<Typography key="successful" color="success.main">
|
||||||
|
{translate("import_users.cards.results.successful", importResults.succeededRecords.length)}
|
||||||
|
</Typography>
|
||||||
|
<List dense>
|
||||||
|
{importResults.succeededRecords.map((record) => (
|
||||||
|
<ListItem key={record.id}>
|
||||||
|
<ListItemText primary={record.displayname} />
|
||||||
|
</ListItem>
|
||||||
|
))}
|
||||||
|
</List>
|
||||||
|
{importResults.skippedRecords.length > 0 && (
|
||||||
|
<Box>
|
||||||
|
<Typography key="skipped" color="warning.main">
|
||||||
|
{translate("import_users.cards.results.skipped", importResults.skippedRecords.length)}
|
||||||
|
</Typography>
|
||||||
|
<Button
|
||||||
|
variant="outlined"
|
||||||
|
startIcon={<DownloadIcon />}
|
||||||
|
onClick={downloadSkippedRecords}
|
||||||
|
sx={{ mt: 2 }}
|
||||||
|
label={translate("import_users.cards.results.download_skipped")}
|
||||||
|
>
|
||||||
|
</Button>
|
||||||
|
</Box>
|
||||||
|
)}
|
||||||
|
{importResults.erroredRecords.length > 0 && (
|
||||||
|
<Typography key="errored" color="error.main">
|
||||||
|
{translate("import_users.cards.results.skipped", importResults.erroredRecords.length)}
|
||||||
|
</Typography>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{importResults.wasDryRun && (
|
||||||
|
<Alert severity="warning" key="simulated">
|
||||||
|
{translate("import_users.cards.results.simulated_only")}
|
||||||
|
</Alert>
|
||||||
|
)}
|
||||||
|
</Stack>
|
||||||
|
</CardContent>
|
||||||
|
</Paper>
|
||||||
|
<Box sx={{ mt: 2 }}>
|
||||||
|
<Link to="/users"><Button variant="outlined" startIcon={<ArrowBackIcon />} label={translate("ra.action.back")} /></Link>
|
||||||
|
</Box>
|
||||||
|
</Container>
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
|
export default ResultsCard;
|
||||||
39
src/components/user-import/StartImportCard.tsx
Normal file
39
src/components/user-import/StartImportCard.tsx
Normal file
@ -0,0 +1,39 @@
|
|||||||
|
import { Button, Checkbox, Paper, Container } from "@mui/material";
|
||||||
|
|
||||||
|
import { CardActions, FormControlLabel } from "@mui/material";
|
||||||
|
import { Progress, ImportLine, ImportResult } from "./types";
|
||||||
|
import { ChangeEventHandler } from "react";
|
||||||
|
import { useTranslate } from "ra-core";
|
||||||
|
|
||||||
|
const StartImportCard = (
|
||||||
|
{ csvData, importResults, progress, dryRun, onDryRunModeChanged, runImport }:
|
||||||
|
{ csvData: ImportLine[], importResults: ImportResult | null, progress: Progress, dryRun: boolean, onDryRunModeChanged: ChangeEventHandler<HTMLInputElement>, runImport: () => void }
|
||||||
|
) => {
|
||||||
|
const translate = useTranslate();
|
||||||
|
if (!csvData || csvData.length === 0 || importResults) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return (
|
||||||
|
<Container>
|
||||||
|
<Paper>
|
||||||
|
<CardActions>
|
||||||
|
<FormControlLabel
|
||||||
|
control={<Checkbox checked={dryRun} onChange={onDryRunModeChanged} disabled={progress !== null} />}
|
||||||
|
label={translate("import_users.cards.startImport.simulate_only")}
|
||||||
|
/>
|
||||||
|
<Button variant="contained" size="large" onClick={runImport} disabled={progress !== null}>
|
||||||
|
{translate("import_users.cards.startImport.run_import")}
|
||||||
|
</Button>
|
||||||
|
{progress !== null ? (
|
||||||
|
<div>
|
||||||
|
{progress.done} of {progress.limit} done
|
||||||
|
</div>
|
||||||
|
) : null}
|
||||||
|
</CardActions>
|
||||||
|
</Paper>
|
||||||
|
</Container>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export default StartImportCard;
|
||||||
83
src/components/user-import/StatsCard.tsx
Normal file
83
src/components/user-import/StatsCard.tsx
Normal file
@ -0,0 +1,83 @@
|
|||||||
|
import { Card, Paper, Stack, CardContent, CardHeader, Container, Typography } from "@mui/material";
|
||||||
|
import { NativeSelect } from "@mui/material";
|
||||||
|
import { FormControlLabel } from "@mui/material";
|
||||||
|
import { Checkbox } from "@mui/material";
|
||||||
|
import { useTranslate } from "ra-core";
|
||||||
|
import { ChangeEventHandler } from "react";
|
||||||
|
import { ParsedStats, Progress } from "./types";
|
||||||
|
|
||||||
|
const StatsCard = ({ stats, progress, importResults, useridMode, passwordMode, onUseridModeChanged, onPasswordModeChange }: { stats: ParsedStats | null, progress: Progress, importResults: any, useridMode: string, passwordMode: boolean, onUseridModeChanged: ChangeEventHandler<HTMLSelectElement>, onPasswordModeChange: ChangeEventHandler<HTMLInputElement> }) => {
|
||||||
|
const translate = useTranslate();
|
||||||
|
|
||||||
|
if (!stats) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (importResults) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return (
|
||||||
|
<>
|
||||||
|
<Container sx={{ mb: 3 }}>
|
||||||
|
<Paper>
|
||||||
|
<Card>
|
||||||
|
<CardHeader
|
||||||
|
title={translate("import_users.cards.importstats.header")}
|
||||||
|
sx={{ borderBottom: 1, borderColor: "divider" }}
|
||||||
|
/>
|
||||||
|
<CardContent>
|
||||||
|
<Stack spacing={1}>
|
||||||
|
<Typography>{translate("import_users.cards.importstats.users_total", stats.total)}</Typography>
|
||||||
|
<Typography>{translate("import_users.cards.importstats.guest_count", stats.is_guest)}</Typography>
|
||||||
|
<Typography>{translate("import_users.cards.importstats.admin_count", stats.admin)}</Typography>
|
||||||
|
</Stack>
|
||||||
|
</CardContent>
|
||||||
|
<CardHeader
|
||||||
|
title={translate("import_users.cards.ids.header")}
|
||||||
|
sx={{ borderBottom: 1, borderColor: "divider" }}
|
||||||
|
/>
|
||||||
|
<CardContent>
|
||||||
|
<Stack spacing={2}>
|
||||||
|
<Typography>
|
||||||
|
{stats.id === stats.total
|
||||||
|
? translate("import_users.cards.ids.all_ids_present")
|
||||||
|
: translate("import_users.cards.ids.count_ids_present", stats.id)}
|
||||||
|
</Typography>
|
||||||
|
{stats.id > 0 && (
|
||||||
|
<NativeSelect onChange={onUseridModeChanged} value={useridMode} disabled={progress !== null}>
|
||||||
|
<option value={"ignore"}>{translate("import_users.cards.ids.mode.ignore")}</option>
|
||||||
|
<option value={"update"}>{translate("import_users.cards.ids.mode.update")}</option>
|
||||||
|
</NativeSelect>
|
||||||
|
)}
|
||||||
|
</Stack>
|
||||||
|
</CardContent>
|
||||||
|
<CardHeader
|
||||||
|
title={translate("import_users.cards.passwords.header")}
|
||||||
|
sx={{ borderBottom: 1, borderColor: "divider" }}
|
||||||
|
/>
|
||||||
|
<CardContent>
|
||||||
|
<Stack spacing={1}>
|
||||||
|
<Typography>
|
||||||
|
{stats.password === stats.total
|
||||||
|
? translate("import_users.cards.passwords.all_passwords_present")
|
||||||
|
: translate("import_users.cards.passwords.count_passwords_present", stats.password)}
|
||||||
|
</Typography>
|
||||||
|
{stats.password > 0 && (
|
||||||
|
<FormControlLabel
|
||||||
|
control={
|
||||||
|
<Checkbox checked={passwordMode} disabled={progress !== null} onChange={onPasswordModeChange} />
|
||||||
|
}
|
||||||
|
label={translate("import_users.cards.passwords.use_passwords")}
|
||||||
|
/>
|
||||||
|
)}
|
||||||
|
</Stack>
|
||||||
|
</CardContent>
|
||||||
|
</Card>
|
||||||
|
</Paper>
|
||||||
|
</Container>
|
||||||
|
</>
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
|
export default StatsCard;
|
||||||
36
src/components/user-import/UploadCard.tsx
Normal file
36
src/components/user-import/UploadCard.tsx
Normal file
@ -0,0 +1,36 @@
|
|||||||
|
import { CardHeader, CardContent, Container, Link, Stack, Typography, Paper } from "@mui/material";
|
||||||
|
|
||||||
|
import { useTranslate } from "ra-core";
|
||||||
|
import { ChangeEventHandler } from "react";
|
||||||
|
import { Progress } from "./types";
|
||||||
|
|
||||||
|
const UploadCard = ({ importResults, onFileChange, progress }: { importResults: any, onFileChange: ChangeEventHandler<HTMLInputElement>, progress: Progress }) => {
|
||||||
|
const translate = useTranslate();
|
||||||
|
if (importResults) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return (
|
||||||
|
<Container sx={{ mb: 3 }}>
|
||||||
|
<Paper elevation={1}>
|
||||||
|
<CardHeader
|
||||||
|
title={translate("import_users.cards.upload.header")}
|
||||||
|
sx={{ borderBottom: 1, borderColor: "divider" }}
|
||||||
|
/>
|
||||||
|
<CardContent>
|
||||||
|
<Stack spacing={2}>
|
||||||
|
<Typography>
|
||||||
|
{translate("import_users.cards.upload.explanation")}
|
||||||
|
<Link href="./data/example.csv" sx={{ ml: 1 }}>
|
||||||
|
example.csv
|
||||||
|
</Link>
|
||||||
|
</Typography>
|
||||||
|
<input type="file" onChange={onFileChange} disabled={progress !== null} />
|
||||||
|
</Stack>
|
||||||
|
</CardContent>
|
||||||
|
</Paper>
|
||||||
|
</Container>
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
|
export default UploadCard;
|
||||||
51
src/components/user-import/UserImport.tsx
Normal file
51
src/components/user-import/UserImport.tsx
Normal file
@ -0,0 +1,51 @@
|
|||||||
|
import {
|
||||||
|
Stack,
|
||||||
|
} from "@mui/material";
|
||||||
|
import { useTranslate } from "ra-core";
|
||||||
|
import { Title } from "react-admin";
|
||||||
|
|
||||||
|
import UploadCard from "./UploadCard";
|
||||||
|
import useImportFile from "./useImportFile";
|
||||||
|
import ErrorsCard from "./ErrorsCard";
|
||||||
|
import ConflictModeCard from "./ConflictModeCard";
|
||||||
|
import StatsCard from "./StatsCard";
|
||||||
|
import StartImportCard from "./StartImportCard";
|
||||||
|
import ResultsCard from "./ResultsCard";
|
||||||
|
|
||||||
|
const UserImport = () => {
|
||||||
|
const {
|
||||||
|
csvData,
|
||||||
|
dryRun,
|
||||||
|
importResults,
|
||||||
|
progress,
|
||||||
|
errors,
|
||||||
|
stats,
|
||||||
|
conflictMode,
|
||||||
|
passwordMode,
|
||||||
|
useridMode,
|
||||||
|
onFileChange,
|
||||||
|
onDryRunModeChanged,
|
||||||
|
runImport,
|
||||||
|
onConflictModeChanged,
|
||||||
|
onPasswordModeChange,
|
||||||
|
onUseridModeChanged,
|
||||||
|
downloadSkippedRecords
|
||||||
|
} = useImportFile();
|
||||||
|
|
||||||
|
const translate = useTranslate();
|
||||||
|
|
||||||
|
return (
|
||||||
|
<Stack spacing={3} mt={3} direction="column">
|
||||||
|
<Title defaultTitle={translate("import_users.title")} />
|
||||||
|
<UploadCard importResults={importResults} onFileChange={onFileChange} progress={progress} />
|
||||||
|
<ErrorsCard errors={errors} />
|
||||||
|
<ConflictModeCard stats={stats} importResults={importResults} conflictMode={conflictMode} onConflictModeChanged={onConflictModeChanged} progress={progress} />
|
||||||
|
<StatsCard stats={stats} progress={progress} importResults={importResults} passwordMode={passwordMode} useridMode={useridMode} onPasswordModeChange={onPasswordModeChange} onUseridModeChanged={onUseridModeChanged} />
|
||||||
|
<StartImportCard csvData={csvData} importResults={importResults} progress={progress} dryRun={dryRun} onDryRunModeChanged={onDryRunModeChanged} runImport={runImport} />
|
||||||
|
<ResultsCard importResults={importResults} downloadSkippedRecords={downloadSkippedRecords} />
|
||||||
|
</Stack>
|
||||||
|
);
|
||||||
|
|
||||||
|
};
|
||||||
|
|
||||||
|
export default UserImport;
|
||||||
47
src/components/user-import/types.ts
Normal file
47
src/components/user-import/types.ts
Normal file
@ -0,0 +1,47 @@
|
|||||||
|
import { RaRecord } from "react-admin";
|
||||||
|
|
||||||
|
export interface ImportLine {
|
||||||
|
id: string;
|
||||||
|
displayname: string;
|
||||||
|
user_type?: string;
|
||||||
|
name?: string;
|
||||||
|
deactivated?: boolean;
|
||||||
|
is_guest?: boolean;
|
||||||
|
admin?: boolean;
|
||||||
|
is_admin?: boolean;
|
||||||
|
password?: string;
|
||||||
|
avatar_url?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ParsedStats {
|
||||||
|
user_types: { [key: string]: number };
|
||||||
|
is_guest: number;
|
||||||
|
admin: number;
|
||||||
|
deactivated: number;
|
||||||
|
password: number;
|
||||||
|
avatar_url: number;
|
||||||
|
id: number;
|
||||||
|
total: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ChangeStats {
|
||||||
|
total: number;
|
||||||
|
id: number;
|
||||||
|
is_guest: number;
|
||||||
|
admin: number;
|
||||||
|
password: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type Progress = {
|
||||||
|
done: number;
|
||||||
|
limit: number;
|
||||||
|
} | null;
|
||||||
|
|
||||||
|
export interface ImportResult {
|
||||||
|
skippedRecords: RaRecord[];
|
||||||
|
erroredRecords: RaRecord[];
|
||||||
|
succeededRecords: RaRecord[];
|
||||||
|
totalRecordCount: number;
|
||||||
|
changeStats: ChangeStats;
|
||||||
|
wasDryRun: boolean;
|
||||||
|
}
|
||||||
377
src/components/user-import/useImportFile.tsx
Normal file
377
src/components/user-import/useImportFile.tsx
Normal file
@ -0,0 +1,377 @@
|
|||||||
|
import { ChangeEvent, useState } from "react";
|
||||||
|
import { useTranslate, useNotify, HttpError } from "react-admin";
|
||||||
|
import { parse as parseCsv, unparse as unparseCsv, ParseResult } from "papaparse";
|
||||||
|
import dataProvider from "../../synapse/dataProvider";
|
||||||
|
import { returnMXID } from "../../utils/mxid";
|
||||||
|
import { generateRandomPassword } from "../../utils/password";
|
||||||
|
import { generateRandomMXID } from "../../utils/mxid";
|
||||||
|
import { ImportLine, ParsedStats, Progress, ImportResult, ChangeStats } from "./types";
|
||||||
|
|
||||||
|
const LOGGING = true;
|
||||||
|
|
||||||
|
const EXPECTED_FIELDS = ["id", "displayname"].sort();
|
||||||
|
|
||||||
|
const useImportFile = () => {
|
||||||
|
const [csvData, setCsvData] = useState<ImportLine[]>([]);
|
||||||
|
const [errors, setErrors] = useState<string[]>([]);
|
||||||
|
const [stats, setStats] = useState<ParsedStats | null>(null);
|
||||||
|
const [dryRun, setDryRun] = useState(true);
|
||||||
|
|
||||||
|
const [progress, setProgress] = useState<Progress>(null);
|
||||||
|
|
||||||
|
const [importResults, setImportResults] = useState<ImportResult | null>(null);
|
||||||
|
const [skippedRecords, setSkippedRecords] = useState<string>("");
|
||||||
|
|
||||||
|
const [conflictMode, setConflictMode] = useState<"stop" | "skip">("stop");
|
||||||
|
const [passwordMode, setPasswordMode] = useState(true);
|
||||||
|
const [useridMode, setUseridMode] = useState<"update" | "ignore">("update");
|
||||||
|
|
||||||
|
const translate = useTranslate();
|
||||||
|
const notify = useNotify();
|
||||||
|
|
||||||
|
const onFileChange = async (e: ChangeEvent<HTMLInputElement>) => {
|
||||||
|
if (progress !== null) return;
|
||||||
|
|
||||||
|
setCsvData([]);
|
||||||
|
setErrors([]);
|
||||||
|
setStats(null);
|
||||||
|
setImportResults(null);
|
||||||
|
const file = e.target.files ? e.target.files[0] : null;
|
||||||
|
if (!file) return;
|
||||||
|
/* Let's refuse some unreasonably big files instead of freezing
|
||||||
|
* up the browser */
|
||||||
|
if (file.size > 100000000) {
|
||||||
|
const message = translate("import_users.errors.unreasonably_big", {
|
||||||
|
size: (file.size / (1024 * 1024)).toFixed(2),
|
||||||
|
});
|
||||||
|
notify(message);
|
||||||
|
setErrors([message]);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
parseCsv<ImportLine>(file, {
|
||||||
|
header: true,
|
||||||
|
skipEmptyLines: true /* especially for a final EOL in the csv file */,
|
||||||
|
complete: result => {
|
||||||
|
if (result.errors) {
|
||||||
|
setErrors(result.errors.map(e => e.toString()));
|
||||||
|
}
|
||||||
|
/* Papaparse is very lenient, we may be able to salvage
|
||||||
|
* the data in the file. */
|
||||||
|
verifyCsv(result);
|
||||||
|
},
|
||||||
|
});
|
||||||
|
} catch {
|
||||||
|
setErrors(["Unknown error"]);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const verifyCsv = ({ data, meta, errors }: ParseResult<ImportLine>) => {
|
||||||
|
/* First, verify the presence of required fields */
|
||||||
|
meta.fields = meta.fields?.map(f => f.trim().toLowerCase());
|
||||||
|
const missingFields = EXPECTED_FIELDS.filter(eF => !meta.fields?.find(mF => eF === mF));
|
||||||
|
|
||||||
|
if (missingFields.length > 0) {
|
||||||
|
setErrors([translate("import_users.error.required_field", { field: missingFields[0] })]);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/* Collect some stats to prevent sneaky csv files from adding admin
|
||||||
|
users or something.
|
||||||
|
*/
|
||||||
|
const stats: ParsedStats = {
|
||||||
|
user_types: { default: 0 },
|
||||||
|
is_guest: 0,
|
||||||
|
admin: 0,
|
||||||
|
deactivated: 0,
|
||||||
|
password: 0,
|
||||||
|
avatar_url: 0,
|
||||||
|
id: 0,
|
||||||
|
|
||||||
|
total: data.length,
|
||||||
|
};
|
||||||
|
|
||||||
|
const errorMessages = errors.map(e => e.message);
|
||||||
|
// sanitize the data first
|
||||||
|
data = data.map(line => {
|
||||||
|
const newLine = {} as ImportLine;
|
||||||
|
for (const [key, value] of Object.entries(line)) {
|
||||||
|
newLine[key.trim().toLowerCase()] = value;
|
||||||
|
}
|
||||||
|
return newLine;
|
||||||
|
});
|
||||||
|
|
||||||
|
// process the data
|
||||||
|
data.forEach((line, idx) => {
|
||||||
|
if (line.user_type === undefined || line.user_type === "") {
|
||||||
|
stats.user_types.default++;
|
||||||
|
} else {
|
||||||
|
stats.user_types[line.user_type] += 1;
|
||||||
|
}
|
||||||
|
/* XXX correct the csv export that react-admin offers for the users
|
||||||
|
* resource so it gives sensible field names and doesn't duplicate
|
||||||
|
* id as "name"?
|
||||||
|
*/
|
||||||
|
if (meta.fields?.includes("name")) {
|
||||||
|
delete line.name;
|
||||||
|
}
|
||||||
|
if (meta.fields?.includes("user_type")) {
|
||||||
|
delete line.user_type;
|
||||||
|
}
|
||||||
|
if (meta.fields?.includes("is_admin")) {
|
||||||
|
delete line.is_admin;
|
||||||
|
}
|
||||||
|
|
||||||
|
["is_guest", "admin", "deactivated"].forEach(f => {
|
||||||
|
if (line[f] === "true") {
|
||||||
|
stats[f]++;
|
||||||
|
line[f] = true; // we need true booleans instead of strings
|
||||||
|
} else {
|
||||||
|
if (line[f] !== "false" && line[f] !== "") {
|
||||||
|
console.log("invalid value", line[f], "for field " + f + " in row " + idx);
|
||||||
|
errorMessages.push(
|
||||||
|
translate("import_users.error.invalid_value", {
|
||||||
|
field: f,
|
||||||
|
row: idx,
|
||||||
|
})
|
||||||
|
);
|
||||||
|
}
|
||||||
|
line[f] = false; // default values to false
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
if (line.password !== undefined && line.password !== "") {
|
||||||
|
stats.password++;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (line.avatar_url !== undefined && line.avatar_url !== "") {
|
||||||
|
stats.avatar_url++;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (line.id !== undefined && line.id !== "") {
|
||||||
|
stats.id++;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
if (errorMessages.length > 0) {
|
||||||
|
setErrors(errorMessages);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
setStats(stats);
|
||||||
|
setCsvData(data);
|
||||||
|
|
||||||
|
return true;
|
||||||
|
};
|
||||||
|
|
||||||
|
const onConflictModeChanged = async (e: ChangeEvent<HTMLSelectElement>) => {
|
||||||
|
if (progress !== null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const value = e.target.value as "stop" | "skip";
|
||||||
|
setConflictMode(value);
|
||||||
|
};
|
||||||
|
|
||||||
|
const onPasswordModeChange = (e: ChangeEvent<HTMLInputElement>) => {
|
||||||
|
if (progress !== null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
setPasswordMode(e.target.checked);
|
||||||
|
};
|
||||||
|
|
||||||
|
const onUseridModeChanged = (e: ChangeEvent<HTMLSelectElement>) => {
|
||||||
|
if (progress !== null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const value = e.target.value as "update" | "ignore";
|
||||||
|
setUseridMode(value);
|
||||||
|
};
|
||||||
|
|
||||||
|
const onDryRunModeChanged = (e: ChangeEvent<HTMLInputElement>) => {
|
||||||
|
if (progress !== null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
setDryRun(e.target.checked);
|
||||||
|
};
|
||||||
|
|
||||||
|
const runImport = async () => {
|
||||||
|
if (progress !== null) {
|
||||||
|
notify("import_users.errors.already_in_progress");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const results = await doImport();
|
||||||
|
setImportResults(results);
|
||||||
|
// offer CSV download of skipped or errored records
|
||||||
|
// (so that the user doesn't have to filter out successful
|
||||||
|
// records manually when fixing stuff in the CSV)
|
||||||
|
setSkippedRecords(unparseCsv(results.skippedRecords));
|
||||||
|
if (LOGGING) console.log("Skipped records:");
|
||||||
|
if (LOGGING) console.log(skippedRecords);
|
||||||
|
};
|
||||||
|
|
||||||
|
const doImport = async (): Promise<ImportResult> => {
|
||||||
|
const skippedRecords: ImportLine[] = [];
|
||||||
|
const erroredRecords: ImportLine[] = [];
|
||||||
|
const succeededRecords: ImportLine[] = [];
|
||||||
|
const changeStats: ChangeStats = {
|
||||||
|
total: 0,
|
||||||
|
id: 0,
|
||||||
|
is_guest: 0,
|
||||||
|
admin: 0,
|
||||||
|
password: 0,
|
||||||
|
};
|
||||||
|
let entriesDone = 0;
|
||||||
|
const entriesCount = csvData.length;
|
||||||
|
try {
|
||||||
|
setProgress({ done: entriesDone, limit: entriesCount });
|
||||||
|
for (const entry of csvData) {
|
||||||
|
const userRecord = { ...entry };
|
||||||
|
// No need to do a bunch of cryptographic random number getting if
|
||||||
|
// we are using neither a generated password nor a generated user id.
|
||||||
|
if (useridMode === "ignore" || userRecord.id === undefined || userRecord.id === "") {
|
||||||
|
userRecord.id = generateRandomMXID();
|
||||||
|
}
|
||||||
|
if (passwordMode === false || entry.password === undefined || entry.password === "") {
|
||||||
|
userRecord.password = generateRandomPassword();
|
||||||
|
}
|
||||||
|
// we want to ensure that the ID is always full MXID, otherwise randomly-generated MXIDs will be in the full
|
||||||
|
// form, but the ones from the CSV will be localpart-only.
|
||||||
|
userRecord.id = returnMXID(userRecord.id);
|
||||||
|
/* TODO record update stats (especially admin no -> yes, deactivated x -> !x, ... */
|
||||||
|
|
||||||
|
/* For these modes we will consider the ID that's in the record.
|
||||||
|
* If the mode is "stop", we will not continue adding more records, and
|
||||||
|
* we will offer information on what was already added and what was
|
||||||
|
* skipped.
|
||||||
|
*
|
||||||
|
* If the mode is "skip", we record the record for later, but don't
|
||||||
|
* send it to the server.
|
||||||
|
*
|
||||||
|
* If the mode is "update", we change fields that are reasonable to
|
||||||
|
* update.
|
||||||
|
* - If the "password mode" is "true" (i.e. "use passwords from csv"):
|
||||||
|
* - if the record has a password
|
||||||
|
* - send the password along with the record
|
||||||
|
* - if the record has no password
|
||||||
|
* - generate a new password
|
||||||
|
* - If the "password mode" is "false"
|
||||||
|
* - never generate a new password to update existing users with
|
||||||
|
*/
|
||||||
|
|
||||||
|
/* We just act as if there are no IDs in the CSV, so every user will be
|
||||||
|
* created anew.
|
||||||
|
* We do a simple retry loop so that an accidental hit on an existing ID
|
||||||
|
* doesn't trip us up.
|
||||||
|
*/
|
||||||
|
if (LOGGING) console.log("will check for existence of record " + JSON.stringify(userRecord));
|
||||||
|
let retries = 0;
|
||||||
|
const submitRecord = async (recordData: ImportLine) => {
|
||||||
|
try {
|
||||||
|
const response = await dataProvider.getOne("users", { id: recordData.id });
|
||||||
|
|
||||||
|
if (LOGGING) console.log("already existed");
|
||||||
|
|
||||||
|
if (conflictMode === "stop") {
|
||||||
|
throw new Error(
|
||||||
|
translate("import_users.error.id_exits", {
|
||||||
|
id: recordData.id,
|
||||||
|
})
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (conflictMode === "skip" || useridMode === "update") {
|
||||||
|
skippedRecords.push(recordData);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const newRecordData = Object.assign({}, recordData, {
|
||||||
|
id: generateRandomMXID(),
|
||||||
|
});
|
||||||
|
retries++;
|
||||||
|
|
||||||
|
if (retries > 512) {
|
||||||
|
console.warn("retry loop got stuck? pathological situation?");
|
||||||
|
skippedRecords.push(recordData);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
await submitRecord(newRecordData);
|
||||||
|
} catch (e) {
|
||||||
|
if (!(e instanceof HttpError) || (e.status && e.status !== 404)) {
|
||||||
|
throw e;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (LOGGING) console.log("OK to create record " + recordData.id + " (" + recordData.displayname + ").");
|
||||||
|
|
||||||
|
if (!dryRun) {
|
||||||
|
await dataProvider.create("users", { data: recordData });
|
||||||
|
}
|
||||||
|
succeededRecords.push(recordData);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
await submitRecord(userRecord);
|
||||||
|
entriesDone++;
|
||||||
|
setProgress({ done: entriesDone, limit: csvData.length });
|
||||||
|
}
|
||||||
|
|
||||||
|
setProgress(null);
|
||||||
|
} catch (e) {
|
||||||
|
setErrors([
|
||||||
|
translate("import_users.error.at_entry", {
|
||||||
|
entry: entriesDone + 1,
|
||||||
|
message: e instanceof Error ? e.message : String(e),
|
||||||
|
})
|
||||||
|
]);
|
||||||
|
setProgress(null);
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
skippedRecords,
|
||||||
|
erroredRecords,
|
||||||
|
succeededRecords,
|
||||||
|
totalRecordCount: entriesCount,
|
||||||
|
changeStats,
|
||||||
|
wasDryRun: dryRun,
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
const downloadSkippedRecords = () => {
|
||||||
|
const element = document.createElement("a");
|
||||||
|
console.log(skippedRecords);
|
||||||
|
const file = new Blob([skippedRecords], {
|
||||||
|
type: "text/comma-separated-values",
|
||||||
|
});
|
||||||
|
element.href = URL.createObjectURL(file);
|
||||||
|
element.download = "skippedRecords.csv";
|
||||||
|
document.body.appendChild(element); // Required for this to work in FireFox
|
||||||
|
element.click();
|
||||||
|
};
|
||||||
|
|
||||||
|
return {
|
||||||
|
csvData,
|
||||||
|
dryRun,
|
||||||
|
onDryRunModeChanged,
|
||||||
|
runImport,
|
||||||
|
progress,
|
||||||
|
importResults,
|
||||||
|
errors,
|
||||||
|
stats,
|
||||||
|
conflictMode,
|
||||||
|
passwordMode,
|
||||||
|
useridMode,
|
||||||
|
onConflictModeChanged,
|
||||||
|
onPasswordModeChange,
|
||||||
|
onUseridModeChanged,
|
||||||
|
onFileChange,
|
||||||
|
downloadSkippedRecords
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export default useImportFile;
|
||||||
@ -91,7 +91,7 @@ const de: SynapseTranslationMessages = {
|
|||||||
goToPdf: "Gehe zum PDF",
|
goToPdf: "Gehe zum PDF",
|
||||||
cards: {
|
cards: {
|
||||||
importstats: {
|
importstats: {
|
||||||
header: "Benutzer importieren",
|
header: "Geparste Benutzer für den Import",
|
||||||
users_total: "%{smart_count} Benutzer in der CSV Datei |||| %{smart_count} Benutzer in der CSV Datei",
|
users_total: "%{smart_count} Benutzer in der CSV Datei |||| %{smart_count} Benutzer in der CSV Datei",
|
||||||
guest_count: "%{smart_count} Gast |||| %{smart_count} Gäste",
|
guest_count: "%{smart_count} Gast |||| %{smart_count} Gäste",
|
||||||
admin_count: "%{smart_count} Server Administrator |||| %{smart_count} Server Administratoren",
|
admin_count: "%{smart_count} Server Administrator |||| %{smart_count} Server Administratoren",
|
||||||
|
|||||||
@ -58,7 +58,7 @@ const en: SynapseTranslationMessages = {
|
|||||||
goToPdf: "Go to PDF",
|
goToPdf: "Go to PDF",
|
||||||
cards: {
|
cards: {
|
||||||
importstats: {
|
importstats: {
|
||||||
header: "Import users",
|
header: "Parsed users for import",
|
||||||
users_total: "%{smart_count} user in CSV file |||| %{smart_count} users in CSV file",
|
users_total: "%{smart_count} user in CSV file |||| %{smart_count} users in CSV file",
|
||||||
guest_count: "%{smart_count} guest |||| %{smart_count} guests",
|
guest_count: "%{smart_count} guest |||| %{smart_count} guests",
|
||||||
admin_count: "%{smart_count} admin |||| %{smart_count} admins",
|
admin_count: "%{smart_count} admin |||| %{smart_count} admins",
|
||||||
|
|||||||
@ -51,7 +51,7 @@ const fa: SynapseTranslationMessages = {
|
|||||||
goToPdf: "رفتن به PDF",
|
goToPdf: "رفتن به PDF",
|
||||||
cards: {
|
cards: {
|
||||||
importstats: {
|
importstats: {
|
||||||
header: "وارد کردن کاربران",
|
header: "کاربران پردازش شده برای وارد کردن",
|
||||||
users_total: "%{smart_count} user in CSV file |||| %{smart_count} users in CSV file",
|
users_total: "%{smart_count} user in CSV file |||| %{smart_count} users in CSV file",
|
||||||
guest_count: "%{smart_count} guest |||| %{smart_count} guests",
|
guest_count: "%{smart_count} guest |||| %{smart_count} guests",
|
||||||
admin_count: "%{smart_count} admin |||| %{smart_count} admins",
|
admin_count: "%{smart_count} admin |||| %{smart_count} admins",
|
||||||
|
|||||||
@ -52,7 +52,7 @@ const fr: SynapseTranslationMessages = {
|
|||||||
goToPdf: "Voir le PDF",
|
goToPdf: "Voir le PDF",
|
||||||
cards: {
|
cards: {
|
||||||
importstats: {
|
importstats: {
|
||||||
header: "Importer des utilisateurs",
|
header: "Utilisateurs analysés pour l'import",
|
||||||
users_total:
|
users_total:
|
||||||
"%{smart_count} utilisateur dans le fichier CSV |||| %{smart_count} utilisateurs dans le fichier CSV",
|
"%{smart_count} utilisateur dans le fichier CSV |||| %{smart_count} utilisateurs dans le fichier CSV",
|
||||||
guest_count: "%{smart_count} visiteur |||| %{smart_count} visiteurs",
|
guest_count: "%{smart_count} visiteur |||| %{smart_count} visiteurs",
|
||||||
|
|||||||
@ -51,7 +51,7 @@ const it: SynapseTranslationMessages = {
|
|||||||
goToPdf: "Vai al PDF",
|
goToPdf: "Vai al PDF",
|
||||||
cards: {
|
cards: {
|
||||||
importstats: {
|
importstats: {
|
||||||
header: "Importa utenti",
|
header: "Utenti analizzati per l'importazione",
|
||||||
users_total: "%{smart_count} utente nel file CSV |||| %{smart_count} utenti nel file CSV",
|
users_total: "%{smart_count} utente nel file CSV |||| %{smart_count} utenti nel file CSV",
|
||||||
guest_count: "%{smart_count} ospite |||| %{smart_count} ospiti",
|
guest_count: "%{smart_count} ospite |||| %{smart_count} ospiti",
|
||||||
admin_count: "%{smart_count} amministratore |||| %{smart_count} amministratori",
|
admin_count: "%{smart_count} amministratore |||| %{smart_count} amministratori",
|
||||||
|
|||||||
@ -83,7 +83,7 @@ const ru: SynapseTranslationMessages = {
|
|||||||
goToPdf: "Перейти к PDF",
|
goToPdf: "Перейти к PDF",
|
||||||
cards: {
|
cards: {
|
||||||
importstats: {
|
importstats: {
|
||||||
header: "Импорт пользователей",
|
header: "Анализированные пользователи для импорта",
|
||||||
users_total:
|
users_total:
|
||||||
"%{smart_count} пользователь в CSV файле |||| %{smart_count} пользователя в CSV файле |||| %{smart_count} пользователей в CSV файле",
|
"%{smart_count} пользователь в CSV файле |||| %{smart_count} пользователя в CSV файле |||| %{smart_count} пользователей в CSV файле",
|
||||||
guest_count: "%{smart_count} гость |||| %{smart_count} гостя |||| %{smart_count} гостей",
|
guest_count: "%{smart_count} гость |||| %{smart_count} гостя |||| %{smart_count} гостей",
|
||||||
|
|||||||
@ -86,7 +86,7 @@ const zh: SynapseTranslationMessages = {
|
|||||||
goToPdf: "转到 PDF",
|
goToPdf: "转到 PDF",
|
||||||
cards: {
|
cards: {
|
||||||
importstats: {
|
importstats: {
|
||||||
header: "导入用户",
|
header: "分析用于导入的用户",
|
||||||
users_total: "%{smart_count} 用户在 CSV 文件中 |||| %{smart_count} 用户在 CSV 文件中",
|
users_total: "%{smart_count} 用户在 CSV 文件中 |||| %{smart_count} 用户在 CSV 文件中",
|
||||||
guest_count: "%{smart_count} 访客 |||| %{smart_count} 访客",
|
guest_count: "%{smart_count} 访客 |||| %{smart_count} 访客",
|
||||||
admin_count: "%{smart_count} 管理员 |||| %{smart_count} 管理员",
|
admin_count: "%{smart_count} 管理员 |||| %{smart_count} 管理员",
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user