Make bulk registration CSV import more user-friendly (#411)
* Refactor CSV import * Finish refactoring * Add types file * update readme
This commit is contained in:
Borislav Pantaleev
committed by
GitHub
GitHub
parent
28043dc2ca
commit
7af2831be8
377
src/components/user-import/useImportFile.tsx
Normal file
377
src/components/user-import/useImportFile.tsx
Normal file
@@ -0,0 +1,377 @@
|
||||
import { ChangeEvent, useState } from "react";
|
||||
import { useTranslate, useNotify, HttpError } from "react-admin";
|
||||
import { parse as parseCsv, unparse as unparseCsv, ParseResult } from "papaparse";
|
||||
import dataProvider from "../../synapse/dataProvider";
|
||||
import { returnMXID } from "../../utils/mxid";
|
||||
import { generateRandomPassword } from "../../utils/password";
|
||||
import { generateRandomMXID } from "../../utils/mxid";
|
||||
import { ImportLine, ParsedStats, Progress, ImportResult, ChangeStats } from "./types";
|
||||
|
||||
const LOGGING = true;
|
||||
|
||||
const EXPECTED_FIELDS = ["id", "displayname"].sort();
|
||||
|
||||
const useImportFile = () => {
|
||||
const [csvData, setCsvData] = useState<ImportLine[]>([]);
|
||||
const [errors, setErrors] = useState<string[]>([]);
|
||||
const [stats, setStats] = useState<ParsedStats | null>(null);
|
||||
const [dryRun, setDryRun] = useState(true);
|
||||
|
||||
const [progress, setProgress] = useState<Progress>(null);
|
||||
|
||||
const [importResults, setImportResults] = useState<ImportResult | null>(null);
|
||||
const [skippedRecords, setSkippedRecords] = useState<string>("");
|
||||
|
||||
const [conflictMode, setConflictMode] = useState<"stop" | "skip">("stop");
|
||||
const [passwordMode, setPasswordMode] = useState(true);
|
||||
const [useridMode, setUseridMode] = useState<"update" | "ignore">("update");
|
||||
|
||||
const translate = useTranslate();
|
||||
const notify = useNotify();
|
||||
|
||||
const onFileChange = async (e: ChangeEvent<HTMLInputElement>) => {
|
||||
if (progress !== null) return;
|
||||
|
||||
setCsvData([]);
|
||||
setErrors([]);
|
||||
setStats(null);
|
||||
setImportResults(null);
|
||||
const file = e.target.files ? e.target.files[0] : null;
|
||||
if (!file) return;
|
||||
/* Let's refuse some unreasonably big files instead of freezing
|
||||
* up the browser */
|
||||
if (file.size > 100000000) {
|
||||
const message = translate("import_users.errors.unreasonably_big", {
|
||||
size: (file.size / (1024 * 1024)).toFixed(2),
|
||||
});
|
||||
notify(message);
|
||||
setErrors([message]);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
parseCsv<ImportLine>(file, {
|
||||
header: true,
|
||||
skipEmptyLines: true /* especially for a final EOL in the csv file */,
|
||||
complete: result => {
|
||||
if (result.errors) {
|
||||
setErrors(result.errors.map(e => e.toString()));
|
||||
}
|
||||
/* Papaparse is very lenient, we may be able to salvage
|
||||
* the data in the file. */
|
||||
verifyCsv(result);
|
||||
},
|
||||
});
|
||||
} catch {
|
||||
setErrors(["Unknown error"]);
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
const verifyCsv = ({ data, meta, errors }: ParseResult<ImportLine>) => {
|
||||
/* First, verify the presence of required fields */
|
||||
meta.fields = meta.fields?.map(f => f.trim().toLowerCase());
|
||||
const missingFields = EXPECTED_FIELDS.filter(eF => !meta.fields?.find(mF => eF === mF));
|
||||
|
||||
if (missingFields.length > 0) {
|
||||
setErrors([translate("import_users.error.required_field", { field: missingFields[0] })]);
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
/* Collect some stats to prevent sneaky csv files from adding admin
|
||||
users or something.
|
||||
*/
|
||||
const stats: ParsedStats = {
|
||||
user_types: { default: 0 },
|
||||
is_guest: 0,
|
||||
admin: 0,
|
||||
deactivated: 0,
|
||||
password: 0,
|
||||
avatar_url: 0,
|
||||
id: 0,
|
||||
|
||||
total: data.length,
|
||||
};
|
||||
|
||||
const errorMessages = errors.map(e => e.message);
|
||||
// sanitize the data first
|
||||
data = data.map(line => {
|
||||
const newLine = {} as ImportLine;
|
||||
for (const [key, value] of Object.entries(line)) {
|
||||
newLine[key.trim().toLowerCase()] = value;
|
||||
}
|
||||
return newLine;
|
||||
});
|
||||
|
||||
// process the data
|
||||
data.forEach((line, idx) => {
|
||||
if (line.user_type === undefined || line.user_type === "") {
|
||||
stats.user_types.default++;
|
||||
} else {
|
||||
stats.user_types[line.user_type] += 1;
|
||||
}
|
||||
/* XXX correct the csv export that react-admin offers for the users
|
||||
* resource so it gives sensible field names and doesn't duplicate
|
||||
* id as "name"?
|
||||
*/
|
||||
if (meta.fields?.includes("name")) {
|
||||
delete line.name;
|
||||
}
|
||||
if (meta.fields?.includes("user_type")) {
|
||||
delete line.user_type;
|
||||
}
|
||||
if (meta.fields?.includes("is_admin")) {
|
||||
delete line.is_admin;
|
||||
}
|
||||
|
||||
["is_guest", "admin", "deactivated"].forEach(f => {
|
||||
if (line[f] === "true") {
|
||||
stats[f]++;
|
||||
line[f] = true; // we need true booleans instead of strings
|
||||
} else {
|
||||
if (line[f] !== "false" && line[f] !== "") {
|
||||
console.log("invalid value", line[f], "for field " + f + " in row " + idx);
|
||||
errorMessages.push(
|
||||
translate("import_users.error.invalid_value", {
|
||||
field: f,
|
||||
row: idx,
|
||||
})
|
||||
);
|
||||
}
|
||||
line[f] = false; // default values to false
|
||||
}
|
||||
});
|
||||
|
||||
if (line.password !== undefined && line.password !== "") {
|
||||
stats.password++;
|
||||
}
|
||||
|
||||
if (line.avatar_url !== undefined && line.avatar_url !== "") {
|
||||
stats.avatar_url++;
|
||||
}
|
||||
|
||||
if (line.id !== undefined && line.id !== "") {
|
||||
stats.id++;
|
||||
}
|
||||
});
|
||||
|
||||
if (errorMessages.length > 0) {
|
||||
setErrors(errorMessages);
|
||||
return false;
|
||||
}
|
||||
|
||||
setStats(stats);
|
||||
setCsvData(data);
|
||||
|
||||
return true;
|
||||
};
|
||||
|
||||
const onConflictModeChanged = async (e: ChangeEvent<HTMLSelectElement>) => {
|
||||
if (progress !== null) {
|
||||
return;
|
||||
}
|
||||
|
||||
const value = e.target.value as "stop" | "skip";
|
||||
setConflictMode(value);
|
||||
};
|
||||
|
||||
const onPasswordModeChange = (e: ChangeEvent<HTMLInputElement>) => {
|
||||
if (progress !== null) {
|
||||
return;
|
||||
}
|
||||
|
||||
setPasswordMode(e.target.checked);
|
||||
};
|
||||
|
||||
const onUseridModeChanged = (e: ChangeEvent<HTMLSelectElement>) => {
|
||||
if (progress !== null) {
|
||||
return;
|
||||
}
|
||||
|
||||
const value = e.target.value as "update" | "ignore";
|
||||
setUseridMode(value);
|
||||
};
|
||||
|
||||
const onDryRunModeChanged = (e: ChangeEvent<HTMLInputElement>) => {
|
||||
if (progress !== null) {
|
||||
return;
|
||||
}
|
||||
setDryRun(e.target.checked);
|
||||
};
|
||||
|
||||
const runImport = async () => {
|
||||
if (progress !== null) {
|
||||
notify("import_users.errors.already_in_progress");
|
||||
return;
|
||||
}
|
||||
|
||||
const results = await doImport();
|
||||
setImportResults(results);
|
||||
// offer CSV download of skipped or errored records
|
||||
// (so that the user doesn't have to filter out successful
|
||||
// records manually when fixing stuff in the CSV)
|
||||
setSkippedRecords(unparseCsv(results.skippedRecords));
|
||||
if (LOGGING) console.log("Skipped records:");
|
||||
if (LOGGING) console.log(skippedRecords);
|
||||
};
|
||||
|
||||
const doImport = async (): Promise<ImportResult> => {
|
||||
const skippedRecords: ImportLine[] = [];
|
||||
const erroredRecords: ImportLine[] = [];
|
||||
const succeededRecords: ImportLine[] = [];
|
||||
const changeStats: ChangeStats = {
|
||||
total: 0,
|
||||
id: 0,
|
||||
is_guest: 0,
|
||||
admin: 0,
|
||||
password: 0,
|
||||
};
|
||||
let entriesDone = 0;
|
||||
const entriesCount = csvData.length;
|
||||
try {
|
||||
setProgress({ done: entriesDone, limit: entriesCount });
|
||||
for (const entry of csvData) {
|
||||
const userRecord = { ...entry };
|
||||
// No need to do a bunch of cryptographic random number getting if
|
||||
// we are using neither a generated password nor a generated user id.
|
||||
if (useridMode === "ignore" || userRecord.id === undefined || userRecord.id === "") {
|
||||
userRecord.id = generateRandomMXID();
|
||||
}
|
||||
if (passwordMode === false || entry.password === undefined || entry.password === "") {
|
||||
userRecord.password = generateRandomPassword();
|
||||
}
|
||||
// we want to ensure that the ID is always full MXID, otherwise randomly-generated MXIDs will be in the full
|
||||
// form, but the ones from the CSV will be localpart-only.
|
||||
userRecord.id = returnMXID(userRecord.id);
|
||||
/* TODO record update stats (especially admin no -> yes, deactivated x -> !x, ... */
|
||||
|
||||
/* For these modes we will consider the ID that's in the record.
|
||||
* If the mode is "stop", we will not continue adding more records, and
|
||||
* we will offer information on what was already added and what was
|
||||
* skipped.
|
||||
*
|
||||
* If the mode is "skip", we record the record for later, but don't
|
||||
* send it to the server.
|
||||
*
|
||||
* If the mode is "update", we change fields that are reasonable to
|
||||
* update.
|
||||
* - If the "password mode" is "true" (i.e. "use passwords from csv"):
|
||||
* - if the record has a password
|
||||
* - send the password along with the record
|
||||
* - if the record has no password
|
||||
* - generate a new password
|
||||
* - If the "password mode" is "false"
|
||||
* - never generate a new password to update existing users with
|
||||
*/
|
||||
|
||||
/* We just act as if there are no IDs in the CSV, so every user will be
|
||||
* created anew.
|
||||
* We do a simple retry loop so that an accidental hit on an existing ID
|
||||
* doesn't trip us up.
|
||||
*/
|
||||
if (LOGGING) console.log("will check for existence of record " + JSON.stringify(userRecord));
|
||||
let retries = 0;
|
||||
const submitRecord = async (recordData: ImportLine) => {
|
||||
try {
|
||||
const response = await dataProvider.getOne("users", { id: recordData.id });
|
||||
|
||||
if (LOGGING) console.log("already existed");
|
||||
|
||||
if (conflictMode === "stop") {
|
||||
throw new Error(
|
||||
translate("import_users.error.id_exits", {
|
||||
id: recordData.id,
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
if (conflictMode === "skip" || useridMode === "update") {
|
||||
skippedRecords.push(recordData);
|
||||
return;
|
||||
}
|
||||
|
||||
const newRecordData = Object.assign({}, recordData, {
|
||||
id: generateRandomMXID(),
|
||||
});
|
||||
retries++;
|
||||
|
||||
if (retries > 512) {
|
||||
console.warn("retry loop got stuck? pathological situation?");
|
||||
skippedRecords.push(recordData);
|
||||
return;
|
||||
}
|
||||
|
||||
await submitRecord(newRecordData);
|
||||
} catch (e) {
|
||||
if (!(e instanceof HttpError) || (e.status && e.status !== 404)) {
|
||||
throw e;
|
||||
}
|
||||
|
||||
if (LOGGING) console.log("OK to create record " + recordData.id + " (" + recordData.displayname + ").");
|
||||
|
||||
if (!dryRun) {
|
||||
await dataProvider.create("users", { data: recordData });
|
||||
}
|
||||
succeededRecords.push(recordData);
|
||||
}
|
||||
};
|
||||
|
||||
await submitRecord(userRecord);
|
||||
entriesDone++;
|
||||
setProgress({ done: entriesDone, limit: csvData.length });
|
||||
}
|
||||
|
||||
setProgress(null);
|
||||
} catch (e) {
|
||||
setErrors([
|
||||
translate("import_users.error.at_entry", {
|
||||
entry: entriesDone + 1,
|
||||
message: e instanceof Error ? e.message : String(e),
|
||||
})
|
||||
]);
|
||||
setProgress(null);
|
||||
}
|
||||
|
||||
return {
|
||||
skippedRecords,
|
||||
erroredRecords,
|
||||
succeededRecords,
|
||||
totalRecordCount: entriesCount,
|
||||
changeStats,
|
||||
wasDryRun: dryRun,
|
||||
};
|
||||
};
|
||||
|
||||
const downloadSkippedRecords = () => {
|
||||
const element = document.createElement("a");
|
||||
console.log(skippedRecords);
|
||||
const file = new Blob([skippedRecords], {
|
||||
type: "text/comma-separated-values",
|
||||
});
|
||||
element.href = URL.createObjectURL(file);
|
||||
element.download = "skippedRecords.csv";
|
||||
document.body.appendChild(element); // Required for this to work in FireFox
|
||||
element.click();
|
||||
};
|
||||
|
||||
return {
|
||||
csvData,
|
||||
dryRun,
|
||||
onDryRunModeChanged,
|
||||
runImport,
|
||||
progress,
|
||||
importResults,
|
||||
errors,
|
||||
stats,
|
||||
conflictMode,
|
||||
passwordMode,
|
||||
useridMode,
|
||||
onConflictModeChanged,
|
||||
onPasswordModeChange,
|
||||
onUseridModeChanged,
|
||||
onFileChange,
|
||||
downloadSkippedRecords
|
||||
}
|
||||
}
|
||||
|
||||
export default useImportFile;
|
||||
Reference in New Issue
Block a user