Make bulk registration CSV import more user-friendly (#411)
* Refactor CSV import * Finish refactoring * Add types file * update readme
This commit is contained in:
Borislav Pantaleev
GitHub
parent
28043dc2ca
commit
7af2831be8
@ -110,6 +110,7 @@ The following changes are already implemented:
|
||||
* 🗑️ [Add "Purge Remote Media" button](https://github.com/etkecc/synapse-admin/pull/237)
|
||||
* [Respect base url (`BASE_PATH` / `vite build --base`) when loading `config.json`](https://github.com/etkecc/synapse-admin/pull/274)
|
||||
* [Add Users' Account Data tab](https://github.com/etkecc/synapse-admin/pull/276)
|
||||
* [Make bulk registration CSV import more user-friendly](https://github.com/etkecc/synapse-admin/pull/411)
|
||||
|
||||
#### exclusive for [etke.cc](https://etke.cc) customers
|
||||
|
||||
|
||||
@ -6,7 +6,7 @@ import { Admin, CustomRoutes, Resource, resolveBrowserLocale } from "react-admin
|
||||
import { Route } from "react-router-dom";
|
||||
|
||||
import AdminLayout from "./components/AdminLayout";
|
||||
import UserImport from "./components/UserImport";
|
||||
import UserImport from "./components/user-import/UserImport";
|
||||
import germanMessages from "./i18n/de";
|
||||
import englishMessages from "./i18n/en";
|
||||
import frenchMessages from "./i18n/fr";
|
||||
|
||||
@ -1,575 +0,0 @@
|
||||
import { parse as parseCsv, unparse as unparseCsv, ParseResult } from "papaparse";
|
||||
import { ChangeEvent, useState } from "react";
|
||||
|
||||
import {
|
||||
Button,
|
||||
Card,
|
||||
CardActions,
|
||||
CardContent,
|
||||
CardHeader,
|
||||
Checkbox,
|
||||
Container,
|
||||
FormControlLabel,
|
||||
NativeSelect,
|
||||
} from "@mui/material";
|
||||
import { DataProvider, useTranslate } from "ra-core";
|
||||
import { useDataProvider, useNotify, RaRecord, Title } from "react-admin";
|
||||
|
||||
import { generateRandomMXID, returnMXID } from "../utils/mxid";
|
||||
import { generateRandomPassword } from "../utils/password";
|
||||
|
||||
const LOGGING = true;
|
||||
|
||||
const expectedFields = ["id", "displayname"].sort();
|
||||
|
||||
function TranslatableOption({ value, text }) {
|
||||
const translate = useTranslate();
|
||||
return <option value={value}>{translate(text)}</option>;
|
||||
}
|
||||
|
||||
type Progress = {
|
||||
done: number;
|
||||
limit: number;
|
||||
} | null;
|
||||
|
||||
interface ImportLine {
|
||||
id: string;
|
||||
displayname: string;
|
||||
user_type?: string;
|
||||
name?: string;
|
||||
deactivated?: boolean;
|
||||
guest?: boolean;
|
||||
admin?: boolean;
|
||||
is_admin?: boolean;
|
||||
password?: string;
|
||||
avatar_url?: string;
|
||||
}
|
||||
|
||||
interface ChangeStats {
|
||||
total: number;
|
||||
id: number;
|
||||
is_guest: number;
|
||||
admin: number;
|
||||
password: number;
|
||||
}
|
||||
|
||||
interface ImportResult {
|
||||
skippedRecords: RaRecord[];
|
||||
erroredRecords: RaRecord[];
|
||||
succeededRecords: RaRecord[];
|
||||
totalRecordCount: number;
|
||||
changeStats: ChangeStats;
|
||||
wasDryRun: boolean;
|
||||
}
|
||||
|
||||
const FilePicker = () => {
|
||||
const [values, setValues] = useState<ImportLine[]>([]);
|
||||
const [error, setError] = useState<string | string[] | null>(null);
|
||||
const [stats, setStats] = useState<ChangeStats | null>(null);
|
||||
const [dryRun, setDryRun] = useState(true);
|
||||
|
||||
const [progress, setProgress] = useState<Progress>(null);
|
||||
|
||||
const [importResults, setImportResults] = useState<ImportResult | null>(null);
|
||||
const [skippedRecords, setSkippedRecords] = useState<string>("");
|
||||
|
||||
const [conflictMode, setConflictMode] = useState("stop");
|
||||
const [passwordMode, setPasswordMode] = useState(true);
|
||||
const [useridMode, setUseridMode] = useState("update");
|
||||
|
||||
const translate = useTranslate();
|
||||
const notify = useNotify();
|
||||
|
||||
const dataProvider = useDataProvider();
|
||||
|
||||
const onFileChange = async (e: ChangeEvent<HTMLInputElement>) => {
|
||||
if (progress !== null) return;
|
||||
|
||||
setValues([]);
|
||||
setError(null);
|
||||
setStats(null);
|
||||
setImportResults(null);
|
||||
const file = e.target.files ? e.target.files[0] : null;
|
||||
if (!file) return;
|
||||
/* Let's refuse some unreasonably big files instead of freezing
|
||||
* up the browser */
|
||||
if (file.size > 100000000) {
|
||||
const message = translate("import_users.errors.unreasonably_big", {
|
||||
size: (file.size / (1024 * 1024)).toFixed(2),
|
||||
});
|
||||
notify(message);
|
||||
setError(message);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
parseCsv<ImportLine>(file, {
|
||||
header: true,
|
||||
skipEmptyLines: true /* especially for a final EOL in the csv file */,
|
||||
complete: result => {
|
||||
if (result.errors) {
|
||||
setError(result.errors.map(e => e.toString()));
|
||||
}
|
||||
/* Papaparse is very lenient, we may be able to salvage
|
||||
* the data in the file. */
|
||||
verifyCsv(result, { setValues, setStats, setError });
|
||||
},
|
||||
});
|
||||
} catch {
|
||||
setError("Unknown error");
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
const verifyCsv = ({ data, meta, errors }: ParseResult<ImportLine>, { setValues, setStats, setError }) => {
|
||||
/* First, verify the presence of required fields */
|
||||
meta.fields = meta.fields?.map(f => f.trim().toLowerCase());
|
||||
const missingFields = expectedFields.filter(eF => !meta.fields?.find(mF => eF === mF));
|
||||
|
||||
if (missingFields.length > 0) {
|
||||
setError(translate("import_users.error.required_field", { field: missingFields[0] }));
|
||||
return false;
|
||||
}
|
||||
|
||||
// XXX after deciding on how "name" and friends should be handled below,
|
||||
// this place will want changes, too.
|
||||
|
||||
/* Collect some stats to prevent sneaky csv files from adding admin
|
||||
users or something.
|
||||
*/
|
||||
const stats = {
|
||||
user_types: { default: 0 },
|
||||
is_guest: 0,
|
||||
admin: 0,
|
||||
deactivated: 0,
|
||||
password: 0,
|
||||
avatar_url: 0,
|
||||
id: 0,
|
||||
|
||||
total: data.length,
|
||||
};
|
||||
|
||||
const errorMessages = errors.map(e => e.message);
|
||||
// sanitize the data first
|
||||
data = data.map(line => {
|
||||
const newLine = {} as ImportLine;
|
||||
for (const [key, value] of Object.entries(line)) {
|
||||
newLine[key.trim().toLowerCase()] = value;
|
||||
}
|
||||
return newLine;
|
||||
});
|
||||
// process the data
|
||||
data.forEach((line, idx) => {
|
||||
if (line.user_type === undefined || line.user_type === "") {
|
||||
stats.user_types.default++;
|
||||
} else {
|
||||
stats.user_types[line.user_type] += 1;
|
||||
}
|
||||
/* XXX correct the csv export that react-admin offers for the users
|
||||
* resource so it gives sensible field names and doesn't duplicate
|
||||
* id as "name"?
|
||||
*/
|
||||
if (meta.fields?.includes("name")) {
|
||||
delete line.name;
|
||||
}
|
||||
if (meta.fields?.includes("user_type")) {
|
||||
delete line.user_type;
|
||||
}
|
||||
if (meta.fields?.includes("is_admin")) {
|
||||
delete line.is_admin;
|
||||
}
|
||||
|
||||
["is_guest", "admin", "deactivated"].forEach(f => {
|
||||
if (line[f] === "true") {
|
||||
stats[f]++;
|
||||
line[f] = true; // we need true booleans instead of strings
|
||||
} else {
|
||||
if (line[f] !== "false" && line[f] !== "") {
|
||||
console.log("invalid value", line[f], "for field " + f + " in row " + idx);
|
||||
errorMessages.push(
|
||||
translate("import_users.error.invalid_value", {
|
||||
field: f,
|
||||
row: idx,
|
||||
})
|
||||
);
|
||||
}
|
||||
line[f] = false; // default values to false
|
||||
}
|
||||
});
|
||||
|
||||
if (line.password !== undefined && line.password !== "") {
|
||||
stats.password++;
|
||||
}
|
||||
|
||||
if (line.avatar_url !== undefined && line.avatar_url !== "") {
|
||||
stats.avatar_url++;
|
||||
}
|
||||
|
||||
if (line.id !== undefined && line.id !== "") {
|
||||
stats.id++;
|
||||
}
|
||||
});
|
||||
|
||||
if (errorMessages.length > 0) {
|
||||
setError(errorMessages);
|
||||
}
|
||||
setStats(stats);
|
||||
setValues(data);
|
||||
|
||||
return true;
|
||||
};
|
||||
|
||||
const runImport = async () => {
|
||||
if (progress !== null) {
|
||||
notify("import_users.errors.already_in_progress");
|
||||
return;
|
||||
}
|
||||
|
||||
const results = await doImport(
|
||||
dataProvider,
|
||||
values,
|
||||
conflictMode,
|
||||
passwordMode,
|
||||
useridMode,
|
||||
dryRun,
|
||||
setProgress,
|
||||
setError
|
||||
);
|
||||
setImportResults(results);
|
||||
// offer CSV download of skipped or errored records
|
||||
// (so that the user doesn't have to filter out successful
|
||||
// records manually when fixing stuff in the CSV)
|
||||
setSkippedRecords(unparseCsv(results.skippedRecords));
|
||||
if (LOGGING) console.log("Skipped records:");
|
||||
if (LOGGING) console.log(skippedRecords);
|
||||
};
|
||||
|
||||
// XXX every single one of the requests will restart the activity indicator
|
||||
// which doesn't look very good.
|
||||
|
||||
const doImport = async (
|
||||
dataProvider: DataProvider,
|
||||
data: ImportLine[],
|
||||
conflictMode: string,
|
||||
passwordMode: boolean,
|
||||
useridMode: string,
|
||||
dryRun: boolean,
|
||||
setProgress: (progress: Progress) => void,
|
||||
setError: (message: string) => void
|
||||
): Promise<ImportResult> => {
|
||||
const skippedRecords: ImportLine[] = [];
|
||||
const erroredRecords: ImportLine[] = [];
|
||||
const succeededRecords: ImportLine[] = [];
|
||||
const changeStats: ChangeStats = {
|
||||
total: 0,
|
||||
id: 0,
|
||||
is_guest: 0,
|
||||
admin: 0,
|
||||
password: 0,
|
||||
};
|
||||
let entriesDone = 0;
|
||||
const entriesCount = data.length;
|
||||
try {
|
||||
setProgress({ done: entriesDone, limit: entriesCount });
|
||||
for (const entry of data) {
|
||||
const userRecord = { ...entry };
|
||||
// No need to do a bunch of cryptographic random number getting if
|
||||
// we are using neither a generated password nor a generated user id.
|
||||
if (useridMode === "ignore" || userRecord.id === undefined || userRecord.id === "") {
|
||||
userRecord.id = generateRandomMXID();
|
||||
}
|
||||
if (passwordMode === false || entry.password === undefined || entry.password === "") {
|
||||
userRecord.password = generateRandomPassword();
|
||||
}
|
||||
// we want to ensure that the ID is always full MXID, otherwise randomly-generated MXIDs will be in the full
|
||||
// form, but the ones from the CSV will be localpart-only.
|
||||
userRecord.id = returnMXID(userRecord.id);
|
||||
/* TODO record update stats (especially admin no -> yes, deactivated x -> !x, ... */
|
||||
|
||||
/* For these modes we will consider the ID that's in the record.
|
||||
* If the mode is "stop", we will not continue adding more records, and
|
||||
* we will offer information on what was already added and what was
|
||||
* skipped.
|
||||
*
|
||||
* If the mode is "skip", we record the record for later, but don't
|
||||
* send it to the server.
|
||||
*
|
||||
* If the mode is "update", we change fields that are reasonable to
|
||||
* update.
|
||||
* - If the "password mode" is "true" (i.e. "use passwords from csv"):
|
||||
* - if the record has a password
|
||||
* - send the password along with the record
|
||||
* - if the record has no password
|
||||
* - generate a new password
|
||||
* - If the "password mode" is "false"
|
||||
* - never generate a new password to update existing users with
|
||||
*/
|
||||
|
||||
/* We just act as if there are no IDs in the CSV, so every user will be
|
||||
* created anew.
|
||||
* We do a simple retry loop so that an accidental hit on an existing ID
|
||||
* doesn't trip us up.
|
||||
*/
|
||||
if (LOGGING) console.log("will check for existence of record " + JSON.stringify(userRecord));
|
||||
let retries = 0;
|
||||
const submitRecord = (recordData: ImportLine) => {
|
||||
return dataProvider.getOne("users", { id: recordData.id }).then(
|
||||
async () => {
|
||||
if (LOGGING) console.log("already existed");
|
||||
|
||||
if (useridMode === "update" || conflictMode === "skip") {
|
||||
skippedRecords.push(recordData);
|
||||
} else if (conflictMode === "stop") {
|
||||
throw new Error(
|
||||
translate("import_users.error.id_exits", {
|
||||
id: recordData.id,
|
||||
})
|
||||
);
|
||||
} else {
|
||||
const newRecordData = Object.assign({}, recordData, {
|
||||
id: generateRandomMXID(),
|
||||
});
|
||||
retries++;
|
||||
if (retries > 512) {
|
||||
console.warn("retry loop got stuck? pathological situation?");
|
||||
skippedRecords.push(recordData);
|
||||
} else {
|
||||
await submitRecord(newRecordData);
|
||||
}
|
||||
}
|
||||
},
|
||||
async () => {
|
||||
if (LOGGING) console.log("OK to create record " + recordData.id + " (" + recordData.displayname + ").");
|
||||
|
||||
if (!dryRun) {
|
||||
await dataProvider.create("users", { data: recordData });
|
||||
}
|
||||
succeededRecords.push(recordData);
|
||||
}
|
||||
);
|
||||
};
|
||||
|
||||
await submitRecord(userRecord);
|
||||
entriesDone++;
|
||||
setProgress({ done: entriesDone, limit: data.length });
|
||||
}
|
||||
|
||||
setProgress(null);
|
||||
} catch (e) {
|
||||
setError(
|
||||
translate("import_users.error.at_entry", {
|
||||
entry: entriesDone + 1,
|
||||
message: e instanceof Error ? e.message : String(e),
|
||||
})
|
||||
);
|
||||
setProgress(null);
|
||||
}
|
||||
return {
|
||||
skippedRecords,
|
||||
erroredRecords,
|
||||
succeededRecords,
|
||||
totalRecordCount: entriesCount,
|
||||
changeStats,
|
||||
wasDryRun: dryRun,
|
||||
};
|
||||
};
|
||||
|
||||
const downloadSkippedRecords = () => {
|
||||
const element = document.createElement("a");
|
||||
console.log(skippedRecords);
|
||||
const file = new Blob([skippedRecords], {
|
||||
type: "text/comma-separated-values",
|
||||
});
|
||||
element.href = URL.createObjectURL(file);
|
||||
element.download = "skippedRecords.csv";
|
||||
document.body.appendChild(element); // Required for this to work in FireFox
|
||||
element.click();
|
||||
};
|
||||
|
||||
const onConflictModeChanged = async (e: ChangeEvent<HTMLSelectElement>) => {
|
||||
if (progress !== null) {
|
||||
return;
|
||||
}
|
||||
|
||||
const value = e.target.value;
|
||||
setConflictMode(value);
|
||||
};
|
||||
|
||||
const onPasswordModeChange = (e: ChangeEvent<HTMLInputElement>) => {
|
||||
if (progress !== null) {
|
||||
return;
|
||||
}
|
||||
|
||||
setPasswordMode(e.target.checked);
|
||||
};
|
||||
|
||||
const onUseridModeChanged = async (e: ChangeEvent<HTMLSelectElement>) => {
|
||||
if (progress !== null) {
|
||||
return;
|
||||
}
|
||||
|
||||
const value = e.target.value;
|
||||
setUseridMode(value);
|
||||
};
|
||||
|
||||
const onDryRunModeChanged = (e: ChangeEvent<HTMLInputElement>) => {
|
||||
if (progress !== null) {
|
||||
return;
|
||||
}
|
||||
setDryRun(e.target.checked);
|
||||
};
|
||||
|
||||
// render individual small components
|
||||
|
||||
const statsCards = stats &&
|
||||
!importResults && [
|
||||
<Container>
|
||||
<CardHeader title={translate("import_users.cards.importstats.header")} />
|
||||
<CardContent>
|
||||
<div>{translate("import_users.cards.importstats.users_total", stats.total)}</div>
|
||||
<div>{translate("import_users.cards.importstats.guest_count", stats.is_guest)}</div>
|
||||
<div>{translate("import_users.cards.importstats.admin_count", stats.admin)}</div>
|
||||
</CardContent>
|
||||
</Container>,
|
||||
<Container>
|
||||
<CardHeader title={translate("import_users.cards.ids.header")} />
|
||||
<CardContent>
|
||||
<div>
|
||||
{stats.id === stats.total
|
||||
? translate("import_users.cards.ids.all_ids_present")
|
||||
: translate("import_users.cards.ids.count_ids_present", stats.id)}
|
||||
</div>
|
||||
{stats.id > 0 ? (
|
||||
<div>
|
||||
<NativeSelect onChange={onUseridModeChanged} value={useridMode} disabled={progress !== null}>
|
||||
<TranslatableOption value="ignore" text="import_users.cards.ids.mode.ignore" />
|
||||
<TranslatableOption value="update" text="import_users.cards.ids.mode.update" />
|
||||
</NativeSelect>
|
||||
</div>
|
||||
) : (
|
||||
""
|
||||
)}
|
||||
</CardContent>
|
||||
</Container>,
|
||||
<Container>
|
||||
<CardHeader title={translate("import_users.cards.passwords.header")} />
|
||||
<CardContent>
|
||||
<div>
|
||||
{stats.password === stats.total
|
||||
? translate("import_users.cards.passwords.all_passwords_present")
|
||||
: translate("import_users.cards.passwords.count_passwords_present", stats.password)}
|
||||
</div>
|
||||
{stats.password > 0 ? (
|
||||
<div>
|
||||
<FormControlLabel
|
||||
control={
|
||||
<Checkbox checked={passwordMode} disabled={progress !== null} onChange={onPasswordModeChange} />
|
||||
}
|
||||
label={translate("import_users.cards.passwords.use_passwords")}
|
||||
/>
|
||||
</div>
|
||||
) : (
|
||||
""
|
||||
)}
|
||||
</CardContent>
|
||||
</Container>,
|
||||
];
|
||||
|
||||
const conflictCards = stats && !importResults && (
|
||||
<Container>
|
||||
<CardHeader title={translate("import_users.cards.conflicts.header")} />
|
||||
<CardContent>
|
||||
<div>
|
||||
<NativeSelect onChange={onConflictModeChanged} value={conflictMode} disabled={progress !== null}>
|
||||
<TranslatableOption value="stop" text="import_users.cards.conflicts.mode.stop" />
|
||||
<TranslatableOption value="skip" text="import_users.cards.conflicts.mode.skip" />
|
||||
</NativeSelect>
|
||||
</div>
|
||||
</CardContent>
|
||||
</Container>
|
||||
);
|
||||
|
||||
const errorCards = error && (
|
||||
<Container>
|
||||
<CardHeader title={translate("import_users.error.error")} />
|
||||
<CardContent>
|
||||
{(Array.isArray(error) ? error : [error]).map(e => (
|
||||
<div>{e}</div>
|
||||
))}
|
||||
</CardContent>
|
||||
</Container>
|
||||
);
|
||||
|
||||
const uploadCard = !importResults && (
|
||||
<Container>
|
||||
<CardHeader title={translate("import_users.cards.upload.header")} />
|
||||
<CardContent>
|
||||
{translate("import_users.cards.upload.explanation")}
|
||||
<a href="./data/example.csv">example.csv</a>
|
||||
<br />
|
||||
<br />
|
||||
<input type="file" onChange={onFileChange} disabled={progress !== null} />
|
||||
</CardContent>
|
||||
</Container>
|
||||
);
|
||||
|
||||
const resultsCard = importResults && (
|
||||
<CardContent>
|
||||
<CardHeader title={translate("import_users.cards.results.header")} />
|
||||
<div>
|
||||
{translate("import_users.cards.results.total", importResults.totalRecordCount)}
|
||||
<br />
|
||||
{translate("import_users.cards.results.successful", importResults.succeededRecords.length)}
|
||||
<br />
|
||||
{importResults.skippedRecords.length
|
||||
? [
|
||||
translate("import_users.cards.results.skipped", importResults.skippedRecords.length),
|
||||
<div>
|
||||
<button onClick={downloadSkippedRecords}>
|
||||
{translate("import_users.cards.results.download_skipped")}
|
||||
</button>
|
||||
</div>,
|
||||
<br />,
|
||||
]
|
||||
: ""}
|
||||
{importResults.erroredRecords.length
|
||||
? [translate("import_users.cards.results.skipped", importResults.erroredRecords.length), <br />]
|
||||
: ""}
|
||||
<br />
|
||||
{importResults.wasDryRun && [translate("import_users.cards.results.simulated_only"), <br />]}
|
||||
</div>
|
||||
</CardContent>
|
||||
);
|
||||
|
||||
const startImportCard =
|
||||
!values || values.length === 0 || importResults ? undefined : (
|
||||
<CardActions>
|
||||
<FormControlLabel
|
||||
control={<Checkbox checked={dryRun} onChange={onDryRunModeChanged} disabled={progress !== null} />}
|
||||
label={translate("import_users.cards.startImport.simulate_only")}
|
||||
/>
|
||||
<Button size="large" onClick={runImport} disabled={progress !== null}>
|
||||
{translate("import_users.cards.startImport.run_import")}
|
||||
</Button>
|
||||
{progress !== null ? (
|
||||
<div>
|
||||
{progress.done} of {progress.limit} done
|
||||
</div>
|
||||
) : null}
|
||||
</CardActions>
|
||||
);
|
||||
|
||||
const allCards: React.JSX.Element[] = [];
|
||||
if (uploadCard) allCards.push(uploadCard);
|
||||
if (errorCards) allCards.push(errorCards);
|
||||
if (conflictCards) allCards.push(conflictCards);
|
||||
if (statsCards) allCards.push(...statsCards);
|
||||
if (startImportCard) allCards.push(startImportCard);
|
||||
if (resultsCard) allCards.push(resultsCard);
|
||||
|
||||
const cardContainer = <Card>{allCards}</Card>;
|
||||
|
||||
return [<Title defaultTitle={translate("import_users.title")} />, cardContainer];
|
||||
};
|
||||
|
||||
export const UserImport = FilePicker;
|
||||
export default UserImport;
|
||||
40
src/components/user-import/ConflictModeCard.tsx
Normal file
40
src/components/user-import/ConflictModeCard.tsx
Normal file
@ -0,0 +1,40 @@
|
||||
import { NativeSelect, Paper } from "@mui/material";
|
||||
|
||||
import { CardContent, CardHeader, Container } from "@mui/material";
|
||||
|
||||
import { useTranslate } from "ra-core";
|
||||
import { ParsedStats, Progress } from "./types";
|
||||
import { ChangeEventHandler } from "react";
|
||||
|
||||
const TranslatableOption = ({ value, text }: { value: string, text: string }) => {
|
||||
const translate = useTranslate();
|
||||
return <option value={value}>{translate(text)}</option>;
|
||||
}
|
||||
|
||||
const ConflictModeCard = ({ stats, importResults, onConflictModeChanged, conflictMode, progress }:
|
||||
{ stats: ParsedStats | null, importResults: any, onConflictModeChanged: ChangeEventHandler<HTMLSelectElement>, conflictMode: string, progress: Progress }) => {
|
||||
const translate = useTranslate();
|
||||
|
||||
if (!stats || importResults) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return (
|
||||
<Container sx={{ mb: 3 }}>
|
||||
<Paper elevation={1}>
|
||||
<CardHeader
|
||||
title={translate("import_users.cards.conflicts.header")}
|
||||
sx={{ borderBottom: 1, borderColor: "divider" }}
|
||||
/>
|
||||
<CardContent>
|
||||
<NativeSelect onChange={onConflictModeChanged} value={conflictMode} disabled={progress !== null}>
|
||||
<TranslatableOption value="stop" text="import_users.cards.conflicts.mode.stop" />
|
||||
<TranslatableOption value="skip" text="import_users.cards.conflicts.mode.skip" />
|
||||
</NativeSelect>
|
||||
</CardContent>
|
||||
</Paper>
|
||||
</Container>
|
||||
);
|
||||
}
|
||||
|
||||
export default ConflictModeCard;
|
||||
43
src/components/user-import/ErrorsCard.tsx
Normal file
43
src/components/user-import/ErrorsCard.tsx
Normal file
@ -0,0 +1,43 @@
|
||||
import {
|
||||
Container,
|
||||
Paper,
|
||||
CardHeader,
|
||||
CardContent,
|
||||
Stack,
|
||||
Typography,
|
||||
} from "@mui/material";
|
||||
import { useTranslate } from "ra-core";
|
||||
|
||||
const ErrorsCard = ({ errors }: { errors: string[] }) => {
|
||||
const translate = useTranslate();
|
||||
|
||||
if (errors.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return (
|
||||
<Container sx={{ mb: 3 }}>
|
||||
<Paper elevation={1}>
|
||||
<CardHeader
|
||||
title={translate("import_users.error.error")}
|
||||
sx={{
|
||||
borderBottom: 1,
|
||||
borderColor: "error.main",
|
||||
color: "error.main"
|
||||
}}
|
||||
/>
|
||||
<CardContent>
|
||||
<Stack spacing={1}>
|
||||
{errors.map((e, idx) => (
|
||||
<Typography key={idx} color="error">
|
||||
{e}
|
||||
</Typography>
|
||||
))}
|
||||
</Stack>
|
||||
</CardContent>
|
||||
</Paper>
|
||||
</Container>
|
||||
);
|
||||
};
|
||||
|
||||
export default ErrorsCard;
|
||||
75
src/components/user-import/ResultsCard.tsx
Normal file
75
src/components/user-import/ResultsCard.tsx
Normal file
@ -0,0 +1,75 @@
|
||||
import { Alert, Box, CardContent, CardHeader, Container, List, ListItem, ListItemText, Paper, Stack, Typography } from "@mui/material"
|
||||
import { Button, Link, useTranslate } from "react-admin";
|
||||
import { ImportResult } from "./types";
|
||||
import DownloadIcon from "@mui/icons-material/Download";
|
||||
import ArrowBackIcon from "@mui/icons-material/ArrowBack";
|
||||
|
||||
const ResultsCard = ({ importResults, downloadSkippedRecords }: { importResults: ImportResult | null, downloadSkippedRecords: () => void }) => {
|
||||
const translate = useTranslate();
|
||||
|
||||
if (!importResults) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return (
|
||||
<Container>
|
||||
<Paper>
|
||||
<CardHeader
|
||||
title={translate("import_users.cards.results.header")}
|
||||
pb={0}
|
||||
sx={{
|
||||
borderBottom: 1,
|
||||
}}
|
||||
/>
|
||||
<CardContent>
|
||||
<Stack spacing={2}>
|
||||
<Typography key="total" color="text.primary">
|
||||
{translate("import_users.cards.results.total", importResults.totalRecordCount)}
|
||||
</Typography>
|
||||
<Typography key="successful" color="success.main">
|
||||
{translate("import_users.cards.results.successful", importResults.succeededRecords.length)}
|
||||
</Typography>
|
||||
<List dense>
|
||||
{importResults.succeededRecords.map((record) => (
|
||||
<ListItem key={record.id}>
|
||||
<ListItemText primary={record.displayname} />
|
||||
</ListItem>
|
||||
))}
|
||||
</List>
|
||||
{importResults.skippedRecords.length > 0 && (
|
||||
<Box>
|
||||
<Typography key="skipped" color="warning.main">
|
||||
{translate("import_users.cards.results.skipped", importResults.skippedRecords.length)}
|
||||
</Typography>
|
||||
<Button
|
||||
variant="outlined"
|
||||
startIcon={<DownloadIcon />}
|
||||
onClick={downloadSkippedRecords}
|
||||
sx={{ mt: 2 }}
|
||||
label={translate("import_users.cards.results.download_skipped")}
|
||||
>
|
||||
</Button>
|
||||
</Box>
|
||||
)}
|
||||
{importResults.erroredRecords.length > 0 && (
|
||||
<Typography key="errored" color="error.main">
|
||||
{translate("import_users.cards.results.skipped", importResults.erroredRecords.length)}
|
||||
</Typography>
|
||||
)}
|
||||
|
||||
{importResults.wasDryRun && (
|
||||
<Alert severity="warning" key="simulated">
|
||||
{translate("import_users.cards.results.simulated_only")}
|
||||
</Alert>
|
||||
)}
|
||||
</Stack>
|
||||
</CardContent>
|
||||
</Paper>
|
||||
<Box sx={{ mt: 2 }}>
|
||||
<Link to="/users"><Button variant="outlined" startIcon={<ArrowBackIcon />} label={translate("ra.action.back")} /></Link>
|
||||
</Box>
|
||||
</Container>
|
||||
);
|
||||
};
|
||||
|
||||
export default ResultsCard;
|
||||
39
src/components/user-import/StartImportCard.tsx
Normal file
39
src/components/user-import/StartImportCard.tsx
Normal file
@ -0,0 +1,39 @@
|
||||
import { Button, Checkbox, Paper, Container } from "@mui/material";
|
||||
|
||||
import { CardActions, FormControlLabel } from "@mui/material";
|
||||
import { Progress, ImportLine, ImportResult } from "./types";
|
||||
import { ChangeEventHandler } from "react";
|
||||
import { useTranslate } from "ra-core";
|
||||
|
||||
const StartImportCard = (
|
||||
{ csvData, importResults, progress, dryRun, onDryRunModeChanged, runImport }:
|
||||
{ csvData: ImportLine[], importResults: ImportResult | null, progress: Progress, dryRun: boolean, onDryRunModeChanged: ChangeEventHandler<HTMLInputElement>, runImport: () => void }
|
||||
) => {
|
||||
const translate = useTranslate();
|
||||
if (!csvData || csvData.length === 0 || importResults) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return (
|
||||
<Container>
|
||||
<Paper>
|
||||
<CardActions>
|
||||
<FormControlLabel
|
||||
control={<Checkbox checked={dryRun} onChange={onDryRunModeChanged} disabled={progress !== null} />}
|
||||
label={translate("import_users.cards.startImport.simulate_only")}
|
||||
/>
|
||||
<Button variant="contained" size="large" onClick={runImport} disabled={progress !== null}>
|
||||
{translate("import_users.cards.startImport.run_import")}
|
||||
</Button>
|
||||
{progress !== null ? (
|
||||
<div>
|
||||
{progress.done} of {progress.limit} done
|
||||
</div>
|
||||
) : null}
|
||||
</CardActions>
|
||||
</Paper>
|
||||
</Container>
|
||||
);
|
||||
}
|
||||
|
||||
export default StartImportCard;
|
||||
83
src/components/user-import/StatsCard.tsx
Normal file
83
src/components/user-import/StatsCard.tsx
Normal file
@ -0,0 +1,83 @@
|
||||
import { Card, Paper, Stack, CardContent, CardHeader, Container, Typography } from "@mui/material";
|
||||
import { NativeSelect } from "@mui/material";
|
||||
import { FormControlLabel } from "@mui/material";
|
||||
import { Checkbox } from "@mui/material";
|
||||
import { useTranslate } from "ra-core";
|
||||
import { ChangeEventHandler } from "react";
|
||||
import { ParsedStats, Progress } from "./types";
|
||||
|
||||
const StatsCard = ({ stats, progress, importResults, useridMode, passwordMode, onUseridModeChanged, onPasswordModeChange }: { stats: ParsedStats | null, progress: Progress, importResults: any, useridMode: string, passwordMode: boolean, onUseridModeChanged: ChangeEventHandler<HTMLSelectElement>, onPasswordModeChange: ChangeEventHandler<HTMLInputElement> }) => {
|
||||
const translate = useTranslate();
|
||||
|
||||
if (!stats) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (importResults) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return (
|
||||
<>
|
||||
<Container sx={{ mb: 3 }}>
|
||||
<Paper>
|
||||
<Card>
|
||||
<CardHeader
|
||||
title={translate("import_users.cards.importstats.header")}
|
||||
sx={{ borderBottom: 1, borderColor: "divider" }}
|
||||
/>
|
||||
<CardContent>
|
||||
<Stack spacing={1}>
|
||||
<Typography>{translate("import_users.cards.importstats.users_total", stats.total)}</Typography>
|
||||
<Typography>{translate("import_users.cards.importstats.guest_count", stats.is_guest)}</Typography>
|
||||
<Typography>{translate("import_users.cards.importstats.admin_count", stats.admin)}</Typography>
|
||||
</Stack>
|
||||
</CardContent>
|
||||
<CardHeader
|
||||
title={translate("import_users.cards.ids.header")}
|
||||
sx={{ borderBottom: 1, borderColor: "divider" }}
|
||||
/>
|
||||
<CardContent>
|
||||
<Stack spacing={2}>
|
||||
<Typography>
|
||||
{stats.id === stats.total
|
||||
? translate("import_users.cards.ids.all_ids_present")
|
||||
: translate("import_users.cards.ids.count_ids_present", stats.id)}
|
||||
</Typography>
|
||||
{stats.id > 0 && (
|
||||
<NativeSelect onChange={onUseridModeChanged} value={useridMode} disabled={progress !== null}>
|
||||
<option value={"ignore"}>{translate("import_users.cards.ids.mode.ignore")}</option>
|
||||
<option value={"update"}>{translate("import_users.cards.ids.mode.update")}</option>
|
||||
</NativeSelect>
|
||||
)}
|
||||
</Stack>
|
||||
</CardContent>
|
||||
<CardHeader
|
||||
title={translate("import_users.cards.passwords.header")}
|
||||
sx={{ borderBottom: 1, borderColor: "divider" }}
|
||||
/>
|
||||
<CardContent>
|
||||
<Stack spacing={1}>
|
||||
<Typography>
|
||||
{stats.password === stats.total
|
||||
? translate("import_users.cards.passwords.all_passwords_present")
|
||||
: translate("import_users.cards.passwords.count_passwords_present", stats.password)}
|
||||
</Typography>
|
||||
{stats.password > 0 && (
|
||||
<FormControlLabel
|
||||
control={
|
||||
<Checkbox checked={passwordMode} disabled={progress !== null} onChange={onPasswordModeChange} />
|
||||
}
|
||||
label={translate("import_users.cards.passwords.use_passwords")}
|
||||
/>
|
||||
)}
|
||||
</Stack>
|
||||
</CardContent>
|
||||
</Card>
|
||||
</Paper>
|
||||
</Container>
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
||||
export default StatsCard;
|
||||
36
src/components/user-import/UploadCard.tsx
Normal file
36
src/components/user-import/UploadCard.tsx
Normal file
@ -0,0 +1,36 @@
|
||||
import { CardHeader, CardContent, Container, Link, Stack, Typography, Paper } from "@mui/material";
|
||||
|
||||
import { useTranslate } from "ra-core";
|
||||
import { ChangeEventHandler } from "react";
|
||||
import { Progress } from "./types";
|
||||
|
||||
const UploadCard = ({ importResults, onFileChange, progress }: { importResults: any, onFileChange: ChangeEventHandler<HTMLInputElement>, progress: Progress }) => {
|
||||
const translate = useTranslate();
|
||||
if (importResults) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return (
|
||||
<Container sx={{ mb: 3 }}>
|
||||
<Paper elevation={1}>
|
||||
<CardHeader
|
||||
title={translate("import_users.cards.upload.header")}
|
||||
sx={{ borderBottom: 1, borderColor: "divider" }}
|
||||
/>
|
||||
<CardContent>
|
||||
<Stack spacing={2}>
|
||||
<Typography>
|
||||
{translate("import_users.cards.upload.explanation")}
|
||||
<Link href="./data/example.csv" sx={{ ml: 1 }}>
|
||||
example.csv
|
||||
</Link>
|
||||
</Typography>
|
||||
<input type="file" onChange={onFileChange} disabled={progress !== null} />
|
||||
</Stack>
|
||||
</CardContent>
|
||||
</Paper>
|
||||
</Container>
|
||||
);
|
||||
};
|
||||
|
||||
export default UploadCard;
|
||||
51
src/components/user-import/UserImport.tsx
Normal file
51
src/components/user-import/UserImport.tsx
Normal file
@ -0,0 +1,51 @@
|
||||
import {
|
||||
Stack,
|
||||
} from "@mui/material";
|
||||
import { useTranslate } from "ra-core";
|
||||
import { Title } from "react-admin";
|
||||
|
||||
import UploadCard from "./UploadCard";
|
||||
import useImportFile from "./useImportFile";
|
||||
import ErrorsCard from "./ErrorsCard";
|
||||
import ConflictModeCard from "./ConflictModeCard";
|
||||
import StatsCard from "./StatsCard";
|
||||
import StartImportCard from "./StartImportCard";
|
||||
import ResultsCard from "./ResultsCard";
|
||||
|
||||
const UserImport = () => {
|
||||
const {
|
||||
csvData,
|
||||
dryRun,
|
||||
importResults,
|
||||
progress,
|
||||
errors,
|
||||
stats,
|
||||
conflictMode,
|
||||
passwordMode,
|
||||
useridMode,
|
||||
onFileChange,
|
||||
onDryRunModeChanged,
|
||||
runImport,
|
||||
onConflictModeChanged,
|
||||
onPasswordModeChange,
|
||||
onUseridModeChanged,
|
||||
downloadSkippedRecords
|
||||
} = useImportFile();
|
||||
|
||||
const translate = useTranslate();
|
||||
|
||||
return (
|
||||
<Stack spacing={3} mt={3} direction="column">
|
||||
<Title defaultTitle={translate("import_users.title")} />
|
||||
<UploadCard importResults={importResults} onFileChange={onFileChange} progress={progress} />
|
||||
<ErrorsCard errors={errors} />
|
||||
<ConflictModeCard stats={stats} importResults={importResults} conflictMode={conflictMode} onConflictModeChanged={onConflictModeChanged} progress={progress} />
|
||||
<StatsCard stats={stats} progress={progress} importResults={importResults} passwordMode={passwordMode} useridMode={useridMode} onPasswordModeChange={onPasswordModeChange} onUseridModeChanged={onUseridModeChanged} />
|
||||
<StartImportCard csvData={csvData} importResults={importResults} progress={progress} dryRun={dryRun} onDryRunModeChanged={onDryRunModeChanged} runImport={runImport} />
|
||||
<ResultsCard importResults={importResults} downloadSkippedRecords={downloadSkippedRecords} />
|
||||
</Stack>
|
||||
);
|
||||
|
||||
};
|
||||
|
||||
export default UserImport;
|
||||
47
src/components/user-import/types.ts
Normal file
47
src/components/user-import/types.ts
Normal file
@ -0,0 +1,47 @@
|
||||
import { RaRecord } from "react-admin";
|
||||
|
||||
export interface ImportLine {
|
||||
id: string;
|
||||
displayname: string;
|
||||
user_type?: string;
|
||||
name?: string;
|
||||
deactivated?: boolean;
|
||||
is_guest?: boolean;
|
||||
admin?: boolean;
|
||||
is_admin?: boolean;
|
||||
password?: string;
|
||||
avatar_url?: string;
|
||||
}
|
||||
|
||||
export interface ParsedStats {
|
||||
user_types: { [key: string]: number };
|
||||
is_guest: number;
|
||||
admin: number;
|
||||
deactivated: number;
|
||||
password: number;
|
||||
avatar_url: number;
|
||||
id: number;
|
||||
total: number;
|
||||
}
|
||||
|
||||
export interface ChangeStats {
|
||||
total: number;
|
||||
id: number;
|
||||
is_guest: number;
|
||||
admin: number;
|
||||
password: number;
|
||||
}
|
||||
|
||||
export type Progress = {
|
||||
done: number;
|
||||
limit: number;
|
||||
} | null;
|
||||
|
||||
export interface ImportResult {
|
||||
skippedRecords: RaRecord[];
|
||||
erroredRecords: RaRecord[];
|
||||
succeededRecords: RaRecord[];
|
||||
totalRecordCount: number;
|
||||
changeStats: ChangeStats;
|
||||
wasDryRun: boolean;
|
||||
}
|
||||
377
src/components/user-import/useImportFile.tsx
Normal file
377
src/components/user-import/useImportFile.tsx
Normal file
@ -0,0 +1,377 @@
|
||||
import { ChangeEvent, useState } from "react";
|
||||
import { useTranslate, useNotify, HttpError } from "react-admin";
|
||||
import { parse as parseCsv, unparse as unparseCsv, ParseResult } from "papaparse";
|
||||
import dataProvider from "../../synapse/dataProvider";
|
||||
import { returnMXID } from "../../utils/mxid";
|
||||
import { generateRandomPassword } from "../../utils/password";
|
||||
import { generateRandomMXID } from "../../utils/mxid";
|
||||
import { ImportLine, ParsedStats, Progress, ImportResult, ChangeStats } from "./types";
|
||||
|
||||
const LOGGING = true;
|
||||
|
||||
const EXPECTED_FIELDS = ["id", "displayname"].sort();
|
||||
|
||||
const useImportFile = () => {
|
||||
const [csvData, setCsvData] = useState<ImportLine[]>([]);
|
||||
const [errors, setErrors] = useState<string[]>([]);
|
||||
const [stats, setStats] = useState<ParsedStats | null>(null);
|
||||
const [dryRun, setDryRun] = useState(true);
|
||||
|
||||
const [progress, setProgress] = useState<Progress>(null);
|
||||
|
||||
const [importResults, setImportResults] = useState<ImportResult | null>(null);
|
||||
const [skippedRecords, setSkippedRecords] = useState<string>("");
|
||||
|
||||
const [conflictMode, setConflictMode] = useState<"stop" | "skip">("stop");
|
||||
const [passwordMode, setPasswordMode] = useState(true);
|
||||
const [useridMode, setUseridMode] = useState<"update" | "ignore">("update");
|
||||
|
||||
const translate = useTranslate();
|
||||
const notify = useNotify();
|
||||
|
||||
const onFileChange = async (e: ChangeEvent<HTMLInputElement>) => {
|
||||
if (progress !== null) return;
|
||||
|
||||
setCsvData([]);
|
||||
setErrors([]);
|
||||
setStats(null);
|
||||
setImportResults(null);
|
||||
const file = e.target.files ? e.target.files[0] : null;
|
||||
if (!file) return;
|
||||
/* Let's refuse some unreasonably big files instead of freezing
|
||||
* up the browser */
|
||||
if (file.size > 100000000) {
|
||||
const message = translate("import_users.errors.unreasonably_big", {
|
||||
size: (file.size / (1024 * 1024)).toFixed(2),
|
||||
});
|
||||
notify(message);
|
||||
setErrors([message]);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
parseCsv<ImportLine>(file, {
|
||||
header: true,
|
||||
skipEmptyLines: true /* especially for a final EOL in the csv file */,
|
||||
complete: result => {
|
||||
if (result.errors) {
|
||||
setErrors(result.errors.map(e => e.toString()));
|
||||
}
|
||||
/* Papaparse is very lenient, we may be able to salvage
|
||||
* the data in the file. */
|
||||
verifyCsv(result);
|
||||
},
|
||||
});
|
||||
} catch {
|
||||
setErrors(["Unknown error"]);
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
const verifyCsv = ({ data, meta, errors }: ParseResult<ImportLine>) => {
|
||||
/* First, verify the presence of required fields */
|
||||
meta.fields = meta.fields?.map(f => f.trim().toLowerCase());
|
||||
const missingFields = EXPECTED_FIELDS.filter(eF => !meta.fields?.find(mF => eF === mF));
|
||||
|
||||
if (missingFields.length > 0) {
|
||||
setErrors([translate("import_users.error.required_field", { field: missingFields[0] })]);
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
/* Collect some stats to prevent sneaky csv files from adding admin
|
||||
users or something.
|
||||
*/
|
||||
const stats: ParsedStats = {
|
||||
user_types: { default: 0 },
|
||||
is_guest: 0,
|
||||
admin: 0,
|
||||
deactivated: 0,
|
||||
password: 0,
|
||||
avatar_url: 0,
|
||||
id: 0,
|
||||
|
||||
total: data.length,
|
||||
};
|
||||
|
||||
const errorMessages = errors.map(e => e.message);
|
||||
// sanitize the data first
|
||||
data = data.map(line => {
|
||||
const newLine = {} as ImportLine;
|
||||
for (const [key, value] of Object.entries(line)) {
|
||||
newLine[key.trim().toLowerCase()] = value;
|
||||
}
|
||||
return newLine;
|
||||
});
|
||||
|
||||
// process the data
|
||||
data.forEach((line, idx) => {
|
||||
if (line.user_type === undefined || line.user_type === "") {
|
||||
stats.user_types.default++;
|
||||
} else {
|
||||
stats.user_types[line.user_type] += 1;
|
||||
}
|
||||
/* XXX correct the csv export that react-admin offers for the users
|
||||
* resource so it gives sensible field names and doesn't duplicate
|
||||
* id as "name"?
|
||||
*/
|
||||
if (meta.fields?.includes("name")) {
|
||||
delete line.name;
|
||||
}
|
||||
if (meta.fields?.includes("user_type")) {
|
||||
delete line.user_type;
|
||||
}
|
||||
if (meta.fields?.includes("is_admin")) {
|
||||
delete line.is_admin;
|
||||
}
|
||||
|
||||
["is_guest", "admin", "deactivated"].forEach(f => {
|
||||
if (line[f] === "true") {
|
||||
stats[f]++;
|
||||
line[f] = true; // we need true booleans instead of strings
|
||||
} else {
|
||||
if (line[f] !== "false" && line[f] !== "") {
|
||||
console.log("invalid value", line[f], "for field " + f + " in row " + idx);
|
||||
errorMessages.push(
|
||||
translate("import_users.error.invalid_value", {
|
||||
field: f,
|
||||
row: idx,
|
||||
})
|
||||
);
|
||||
}
|
||||
line[f] = false; // default values to false
|
||||
}
|
||||
});
|
||||
|
||||
if (line.password !== undefined && line.password !== "") {
|
||||
stats.password++;
|
||||
}
|
||||
|
||||
if (line.avatar_url !== undefined && line.avatar_url !== "") {
|
||||
stats.avatar_url++;
|
||||
}
|
||||
|
||||
if (line.id !== undefined && line.id !== "") {
|
||||
stats.id++;
|
||||
}
|
||||
});
|
||||
|
||||
if (errorMessages.length > 0) {
|
||||
setErrors(errorMessages);
|
||||
return false;
|
||||
}
|
||||
|
||||
setStats(stats);
|
||||
setCsvData(data);
|
||||
|
||||
return true;
|
||||
};
|
||||
|
||||
const onConflictModeChanged = async (e: ChangeEvent<HTMLSelectElement>) => {
|
||||
if (progress !== null) {
|
||||
return;
|
||||
}
|
||||
|
||||
const value = e.target.value as "stop" | "skip";
|
||||
setConflictMode(value);
|
||||
};
|
||||
|
||||
const onPasswordModeChange = (e: ChangeEvent<HTMLInputElement>) => {
|
||||
if (progress !== null) {
|
||||
return;
|
||||
}
|
||||
|
||||
setPasswordMode(e.target.checked);
|
||||
};
|
||||
|
||||
const onUseridModeChanged = (e: ChangeEvent<HTMLSelectElement>) => {
|
||||
if (progress !== null) {
|
||||
return;
|
||||
}
|
||||
|
||||
const value = e.target.value as "update" | "ignore";
|
||||
setUseridMode(value);
|
||||
};
|
||||
|
||||
const onDryRunModeChanged = (e: ChangeEvent<HTMLInputElement>) => {
|
||||
if (progress !== null) {
|
||||
return;
|
||||
}
|
||||
setDryRun(e.target.checked);
|
||||
};
|
||||
|
||||
const runImport = async () => {
|
||||
if (progress !== null) {
|
||||
notify("import_users.errors.already_in_progress");
|
||||
return;
|
||||
}
|
||||
|
||||
const results = await doImport();
|
||||
setImportResults(results);
|
||||
// offer CSV download of skipped or errored records
|
||||
// (so that the user doesn't have to filter out successful
|
||||
// records manually when fixing stuff in the CSV)
|
||||
setSkippedRecords(unparseCsv(results.skippedRecords));
|
||||
if (LOGGING) console.log("Skipped records:");
|
||||
if (LOGGING) console.log(skippedRecords);
|
||||
};
|
||||
|
||||
const doImport = async (): Promise<ImportResult> => {
|
||||
const skippedRecords: ImportLine[] = [];
|
||||
const erroredRecords: ImportLine[] = [];
|
||||
const succeededRecords: ImportLine[] = [];
|
||||
const changeStats: ChangeStats = {
|
||||
total: 0,
|
||||
id: 0,
|
||||
is_guest: 0,
|
||||
admin: 0,
|
||||
password: 0,
|
||||
};
|
||||
let entriesDone = 0;
|
||||
const entriesCount = csvData.length;
|
||||
try {
|
||||
setProgress({ done: entriesDone, limit: entriesCount });
|
||||
for (const entry of csvData) {
|
||||
const userRecord = { ...entry };
|
||||
// No need to do a bunch of cryptographic random number getting if
|
||||
// we are using neither a generated password nor a generated user id.
|
||||
if (useridMode === "ignore" || userRecord.id === undefined || userRecord.id === "") {
|
||||
userRecord.id = generateRandomMXID();
|
||||
}
|
||||
if (passwordMode === false || entry.password === undefined || entry.password === "") {
|
||||
userRecord.password = generateRandomPassword();
|
||||
}
|
||||
// we want to ensure that the ID is always full MXID, otherwise randomly-generated MXIDs will be in the full
|
||||
// form, but the ones from the CSV will be localpart-only.
|
||||
userRecord.id = returnMXID(userRecord.id);
|
||||
/* TODO record update stats (especially admin no -> yes, deactivated x -> !x, ... */
|
||||
|
||||
/* For these modes we will consider the ID that's in the record.
|
||||
* If the mode is "stop", we will not continue adding more records, and
|
||||
* we will offer information on what was already added and what was
|
||||
* skipped.
|
||||
*
|
||||
* If the mode is "skip", we record the record for later, but don't
|
||||
* send it to the server.
|
||||
*
|
||||
* If the mode is "update", we change fields that are reasonable to
|
||||
* update.
|
||||
* - If the "password mode" is "true" (i.e. "use passwords from csv"):
|
||||
* - if the record has a password
|
||||
* - send the password along with the record
|
||||
* - if the record has no password
|
||||
* - generate a new password
|
||||
* - If the "password mode" is "false"
|
||||
* - never generate a new password to update existing users with
|
||||
*/
|
||||
|
||||
/* We just act as if there are no IDs in the CSV, so every user will be
|
||||
* created anew.
|
||||
* We do a simple retry loop so that an accidental hit on an existing ID
|
||||
* doesn't trip us up.
|
||||
*/
|
||||
if (LOGGING) console.log("will check for existence of record " + JSON.stringify(userRecord));
|
||||
let retries = 0;
|
||||
const submitRecord = async (recordData: ImportLine) => {
|
||||
try {
|
||||
const response = await dataProvider.getOne("users", { id: recordData.id });
|
||||
|
||||
if (LOGGING) console.log("already existed");
|
||||
|
||||
if (conflictMode === "stop") {
|
||||
throw new Error(
|
||||
translate("import_users.error.id_exits", {
|
||||
id: recordData.id,
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
if (conflictMode === "skip" || useridMode === "update") {
|
||||
skippedRecords.push(recordData);
|
||||
return;
|
||||
}
|
||||
|
||||
const newRecordData = Object.assign({}, recordData, {
|
||||
id: generateRandomMXID(),
|
||||
});
|
||||
retries++;
|
||||
|
||||
if (retries > 512) {
|
||||
console.warn("retry loop got stuck? pathological situation?");
|
||||
skippedRecords.push(recordData);
|
||||
return;
|
||||
}
|
||||
|
||||
await submitRecord(newRecordData);
|
||||
} catch (e) {
|
||||
if (!(e instanceof HttpError) || (e.status && e.status !== 404)) {
|
||||
throw e;
|
||||
}
|
||||
|
||||
if (LOGGING) console.log("OK to create record " + recordData.id + " (" + recordData.displayname + ").");
|
||||
|
||||
if (!dryRun) {
|
||||
await dataProvider.create("users", { data: recordData });
|
||||
}
|
||||
succeededRecords.push(recordData);
|
||||
}
|
||||
};
|
||||
|
||||
await submitRecord(userRecord);
|
||||
entriesDone++;
|
||||
setProgress({ done: entriesDone, limit: csvData.length });
|
||||
}
|
||||
|
||||
setProgress(null);
|
||||
} catch (e) {
|
||||
setErrors([
|
||||
translate("import_users.error.at_entry", {
|
||||
entry: entriesDone + 1,
|
||||
message: e instanceof Error ? e.message : String(e),
|
||||
})
|
||||
]);
|
||||
setProgress(null);
|
||||
}
|
||||
|
||||
return {
|
||||
skippedRecords,
|
||||
erroredRecords,
|
||||
succeededRecords,
|
||||
totalRecordCount: entriesCount,
|
||||
changeStats,
|
||||
wasDryRun: dryRun,
|
||||
};
|
||||
};
|
||||
|
||||
const downloadSkippedRecords = () => {
|
||||
const element = document.createElement("a");
|
||||
console.log(skippedRecords);
|
||||
const file = new Blob([skippedRecords], {
|
||||
type: "text/comma-separated-values",
|
||||
});
|
||||
element.href = URL.createObjectURL(file);
|
||||
element.download = "skippedRecords.csv";
|
||||
document.body.appendChild(element); // Required for this to work in FireFox
|
||||
element.click();
|
||||
};
|
||||
|
||||
return {
|
||||
csvData,
|
||||
dryRun,
|
||||
onDryRunModeChanged,
|
||||
runImport,
|
||||
progress,
|
||||
importResults,
|
||||
errors,
|
||||
stats,
|
||||
conflictMode,
|
||||
passwordMode,
|
||||
useridMode,
|
||||
onConflictModeChanged,
|
||||
onPasswordModeChange,
|
||||
onUseridModeChanged,
|
||||
onFileChange,
|
||||
downloadSkippedRecords
|
||||
}
|
||||
}
|
||||
|
||||
export default useImportFile;
|
||||
@ -91,7 +91,7 @@ const de: SynapseTranslationMessages = {
|
||||
goToPdf: "Gehe zum PDF",
|
||||
cards: {
|
||||
importstats: {
|
||||
header: "Benutzer importieren",
|
||||
header: "Geparste Benutzer für den Import",
|
||||
users_total: "%{smart_count} Benutzer in der CSV Datei |||| %{smart_count} Benutzer in der CSV Datei",
|
||||
guest_count: "%{smart_count} Gast |||| %{smart_count} Gäste",
|
||||
admin_count: "%{smart_count} Server Administrator |||| %{smart_count} Server Administratoren",
|
||||
|
||||
@ -58,7 +58,7 @@ const en: SynapseTranslationMessages = {
|
||||
goToPdf: "Go to PDF",
|
||||
cards: {
|
||||
importstats: {
|
||||
header: "Import users",
|
||||
header: "Parsed users for import",
|
||||
users_total: "%{smart_count} user in CSV file |||| %{smart_count} users in CSV file",
|
||||
guest_count: "%{smart_count} guest |||| %{smart_count} guests",
|
||||
admin_count: "%{smart_count} admin |||| %{smart_count} admins",
|
||||
|
||||
@ -51,7 +51,7 @@ const fa: SynapseTranslationMessages = {
|
||||
goToPdf: "رفتن به PDF",
|
||||
cards: {
|
||||
importstats: {
|
||||
header: "وارد کردن کاربران",
|
||||
header: "کاربران پردازش شده برای وارد کردن",
|
||||
users_total: "%{smart_count} user in CSV file |||| %{smart_count} users in CSV file",
|
||||
guest_count: "%{smart_count} guest |||| %{smart_count} guests",
|
||||
admin_count: "%{smart_count} admin |||| %{smart_count} admins",
|
||||
|
||||
@ -52,7 +52,7 @@ const fr: SynapseTranslationMessages = {
|
||||
goToPdf: "Voir le PDF",
|
||||
cards: {
|
||||
importstats: {
|
||||
header: "Importer des utilisateurs",
|
||||
header: "Utilisateurs analysés pour l'import",
|
||||
users_total:
|
||||
"%{smart_count} utilisateur dans le fichier CSV |||| %{smart_count} utilisateurs dans le fichier CSV",
|
||||
guest_count: "%{smart_count} visiteur |||| %{smart_count} visiteurs",
|
||||
|
||||
@ -51,7 +51,7 @@ const it: SynapseTranslationMessages = {
|
||||
goToPdf: "Vai al PDF",
|
||||
cards: {
|
||||
importstats: {
|
||||
header: "Importa utenti",
|
||||
header: "Utenti analizzati per l'importazione",
|
||||
users_total: "%{smart_count} utente nel file CSV |||| %{smart_count} utenti nel file CSV",
|
||||
guest_count: "%{smart_count} ospite |||| %{smart_count} ospiti",
|
||||
admin_count: "%{smart_count} amministratore |||| %{smart_count} amministratori",
|
||||
|
||||
@ -83,7 +83,7 @@ const ru: SynapseTranslationMessages = {
|
||||
goToPdf: "Перейти к PDF",
|
||||
cards: {
|
||||
importstats: {
|
||||
header: "Импорт пользователей",
|
||||
header: "Анализированные пользователи для импорта",
|
||||
users_total:
|
||||
"%{smart_count} пользователь в CSV файле |||| %{smart_count} пользователя в CSV файле |||| %{smart_count} пользователей в CSV файле",
|
||||
guest_count: "%{smart_count} гость |||| %{smart_count} гостя |||| %{smart_count} гостей",
|
||||
|
||||
@ -86,7 +86,7 @@ const zh: SynapseTranslationMessages = {
|
||||
goToPdf: "转到 PDF",
|
||||
cards: {
|
||||
importstats: {
|
||||
header: "导入用户",
|
||||
header: "分析用于导入的用户",
|
||||
users_total: "%{smart_count} 用户在 CSV 文件中 |||| %{smart_count} 用户在 CSV 文件中",
|
||||
guest_count: "%{smart_count} 访客 |||| %{smart_count} 访客",
|
||||
admin_count: "%{smart_count} 管理员 |||| %{smart_count} 管理员",
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user