diff --git a/README.md b/README.md index 40300d4..9cf0ba2 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,7 @@ The following changes are already implemented: * 🗑️ [Add "Purge Remote Media" button](https://github.com/etkecc/synapse-admin/pull/237) * [Respect base url (`BASE_PATH` / `vite build --base`) when loading `config.json`](https://github.com/etkecc/synapse-admin/pull/274) * [Add Users' Account Data tab](https://github.com/etkecc/synapse-admin/pull/276) +* [Make bulk registration CSV import more user-friendly](https://github.com/etkecc/synapse-admin/pull/411) #### exclusive for [etke.cc](https://etke.cc) customers diff --git a/src/App.tsx b/src/App.tsx index 7bfd4c7..2aa2efa 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -6,7 +6,7 @@ import { Admin, CustomRoutes, Resource, resolveBrowserLocale } from "react-admin import { Route } from "react-router-dom"; import AdminLayout from "./components/AdminLayout"; -import UserImport from "./components/UserImport"; +import UserImport from "./components/user-import/UserImport"; import germanMessages from "./i18n/de"; import englishMessages from "./i18n/en"; import frenchMessages from "./i18n/fr"; diff --git a/src/components/UserImport.tsx b/src/components/UserImport.tsx deleted file mode 100644 index 244268b..0000000 --- a/src/components/UserImport.tsx +++ /dev/null @@ -1,575 +0,0 @@ -import { parse as parseCsv, unparse as unparseCsv, ParseResult } from "papaparse"; -import { ChangeEvent, useState } from "react"; - -import { - Button, - Card, - CardActions, - CardContent, - CardHeader, - Checkbox, - Container, - FormControlLabel, - NativeSelect, -} from "@mui/material"; -import { DataProvider, useTranslate } from "ra-core"; -import { useDataProvider, useNotify, RaRecord, Title } from "react-admin"; - -import { generateRandomMXID, returnMXID } from "../utils/mxid"; -import { generateRandomPassword } from "../utils/password"; - -const LOGGING = true; - -const expectedFields = ["id", "displayname"].sort(); - -function TranslatableOption({ value, text }) { - const translate = useTranslate(); - return ; -} - -type Progress = { - done: number; - limit: number; -} | null; - -interface ImportLine { - id: string; - displayname: string; - user_type?: string; - name?: string; - deactivated?: boolean; - guest?: boolean; - admin?: boolean; - is_admin?: boolean; - password?: string; - avatar_url?: string; -} - -interface ChangeStats { - total: number; - id: number; - is_guest: number; - admin: number; - password: number; -} - -interface ImportResult { - skippedRecords: RaRecord[]; - erroredRecords: RaRecord[]; - succeededRecords: RaRecord[]; - totalRecordCount: number; - changeStats: ChangeStats; - wasDryRun: boolean; -} - -const FilePicker = () => { - const [values, setValues] = useState([]); - const [error, setError] = useState(null); - const [stats, setStats] = useState(null); - const [dryRun, setDryRun] = useState(true); - - const [progress, setProgress] = useState(null); - - const [importResults, setImportResults] = useState(null); - const [skippedRecords, setSkippedRecords] = useState(""); - - const [conflictMode, setConflictMode] = useState("stop"); - const [passwordMode, setPasswordMode] = useState(true); - const [useridMode, setUseridMode] = useState("update"); - - const translate = useTranslate(); - const notify = useNotify(); - - const dataProvider = useDataProvider(); - - const onFileChange = async (e: ChangeEvent) => { - if (progress !== null) return; - - setValues([]); - setError(null); - setStats(null); - setImportResults(null); - const file = e.target.files ? e.target.files[0] : null; - if (!file) return; - /* Let's refuse some unreasonably big files instead of freezing - * up the browser */ - if (file.size > 100000000) { - const message = translate("import_users.errors.unreasonably_big", { - size: (file.size / (1024 * 1024)).toFixed(2), - }); - notify(message); - setError(message); - return; - } - try { - parseCsv(file, { - header: true, - skipEmptyLines: true /* especially for a final EOL in the csv file */, - complete: result => { - if (result.errors) { - setError(result.errors.map(e => e.toString())); - } - /* Papaparse is very lenient, we may be able to salvage - * the data in the file. */ - verifyCsv(result, { setValues, setStats, setError }); - }, - }); - } catch { - setError("Unknown error"); - return null; - } - }; - - const verifyCsv = ({ data, meta, errors }: ParseResult, { setValues, setStats, setError }) => { - /* First, verify the presence of required fields */ - meta.fields = meta.fields?.map(f => f.trim().toLowerCase()); - const missingFields = expectedFields.filter(eF => !meta.fields?.find(mF => eF === mF)); - - if (missingFields.length > 0) { - setError(translate("import_users.error.required_field", { field: missingFields[0] })); - return false; - } - - // XXX after deciding on how "name" and friends should be handled below, - // this place will want changes, too. - - /* Collect some stats to prevent sneaky csv files from adding admin - users or something. - */ - const stats = { - user_types: { default: 0 }, - is_guest: 0, - admin: 0, - deactivated: 0, - password: 0, - avatar_url: 0, - id: 0, - - total: data.length, - }; - - const errorMessages = errors.map(e => e.message); - // sanitize the data first - data = data.map(line => { - const newLine = {} as ImportLine; - for (const [key, value] of Object.entries(line)) { - newLine[key.trim().toLowerCase()] = value; - } - return newLine; - }); - // process the data - data.forEach((line, idx) => { - if (line.user_type === undefined || line.user_type === "") { - stats.user_types.default++; - } else { - stats.user_types[line.user_type] += 1; - } - /* XXX correct the csv export that react-admin offers for the users - * resource so it gives sensible field names and doesn't duplicate - * id as "name"? - */ - if (meta.fields?.includes("name")) { - delete line.name; - } - if (meta.fields?.includes("user_type")) { - delete line.user_type; - } - if (meta.fields?.includes("is_admin")) { - delete line.is_admin; - } - - ["is_guest", "admin", "deactivated"].forEach(f => { - if (line[f] === "true") { - stats[f]++; - line[f] = true; // we need true booleans instead of strings - } else { - if (line[f] !== "false" && line[f] !== "") { - console.log("invalid value", line[f], "for field " + f + " in row " + idx); - errorMessages.push( - translate("import_users.error.invalid_value", { - field: f, - row: idx, - }) - ); - } - line[f] = false; // default values to false - } - }); - - if (line.password !== undefined && line.password !== "") { - stats.password++; - } - - if (line.avatar_url !== undefined && line.avatar_url !== "") { - stats.avatar_url++; - } - - if (line.id !== undefined && line.id !== "") { - stats.id++; - } - }); - - if (errorMessages.length > 0) { - setError(errorMessages); - } - setStats(stats); - setValues(data); - - return true; - }; - - const runImport = async () => { - if (progress !== null) { - notify("import_users.errors.already_in_progress"); - return; - } - - const results = await doImport( - dataProvider, - values, - conflictMode, - passwordMode, - useridMode, - dryRun, - setProgress, - setError - ); - setImportResults(results); - // offer CSV download of skipped or errored records - // (so that the user doesn't have to filter out successful - // records manually when fixing stuff in the CSV) - setSkippedRecords(unparseCsv(results.skippedRecords)); - if (LOGGING) console.log("Skipped records:"); - if (LOGGING) console.log(skippedRecords); - }; - - // XXX every single one of the requests will restart the activity indicator - // which doesn't look very good. - - const doImport = async ( - dataProvider: DataProvider, - data: ImportLine[], - conflictMode: string, - passwordMode: boolean, - useridMode: string, - dryRun: boolean, - setProgress: (progress: Progress) => void, - setError: (message: string) => void - ): Promise => { - const skippedRecords: ImportLine[] = []; - const erroredRecords: ImportLine[] = []; - const succeededRecords: ImportLine[] = []; - const changeStats: ChangeStats = { - total: 0, - id: 0, - is_guest: 0, - admin: 0, - password: 0, - }; - let entriesDone = 0; - const entriesCount = data.length; - try { - setProgress({ done: entriesDone, limit: entriesCount }); - for (const entry of data) { - const userRecord = { ...entry }; - // No need to do a bunch of cryptographic random number getting if - // we are using neither a generated password nor a generated user id. - if (useridMode === "ignore" || userRecord.id === undefined || userRecord.id === "") { - userRecord.id = generateRandomMXID(); - } - if (passwordMode === false || entry.password === undefined || entry.password === "") { - userRecord.password = generateRandomPassword(); - } - // we want to ensure that the ID is always full MXID, otherwise randomly-generated MXIDs will be in the full - // form, but the ones from the CSV will be localpart-only. - userRecord.id = returnMXID(userRecord.id); - /* TODO record update stats (especially admin no -> yes, deactivated x -> !x, ... */ - - /* For these modes we will consider the ID that's in the record. - * If the mode is "stop", we will not continue adding more records, and - * we will offer information on what was already added and what was - * skipped. - * - * If the mode is "skip", we record the record for later, but don't - * send it to the server. - * - * If the mode is "update", we change fields that are reasonable to - * update. - * - If the "password mode" is "true" (i.e. "use passwords from csv"): - * - if the record has a password - * - send the password along with the record - * - if the record has no password - * - generate a new password - * - If the "password mode" is "false" - * - never generate a new password to update existing users with - */ - - /* We just act as if there are no IDs in the CSV, so every user will be - * created anew. - * We do a simple retry loop so that an accidental hit on an existing ID - * doesn't trip us up. - */ - if (LOGGING) console.log("will check for existence of record " + JSON.stringify(userRecord)); - let retries = 0; - const submitRecord = (recordData: ImportLine) => { - return dataProvider.getOne("users", { id: recordData.id }).then( - async () => { - if (LOGGING) console.log("already existed"); - - if (useridMode === "update" || conflictMode === "skip") { - skippedRecords.push(recordData); - } else if (conflictMode === "stop") { - throw new Error( - translate("import_users.error.id_exits", { - id: recordData.id, - }) - ); - } else { - const newRecordData = Object.assign({}, recordData, { - id: generateRandomMXID(), - }); - retries++; - if (retries > 512) { - console.warn("retry loop got stuck? pathological situation?"); - skippedRecords.push(recordData); - } else { - await submitRecord(newRecordData); - } - } - }, - async () => { - if (LOGGING) console.log("OK to create record " + recordData.id + " (" + recordData.displayname + ")."); - - if (!dryRun) { - await dataProvider.create("users", { data: recordData }); - } - succeededRecords.push(recordData); - } - ); - }; - - await submitRecord(userRecord); - entriesDone++; - setProgress({ done: entriesDone, limit: data.length }); - } - - setProgress(null); - } catch (e) { - setError( - translate("import_users.error.at_entry", { - entry: entriesDone + 1, - message: e instanceof Error ? e.message : String(e), - }) - ); - setProgress(null); - } - return { - skippedRecords, - erroredRecords, - succeededRecords, - totalRecordCount: entriesCount, - changeStats, - wasDryRun: dryRun, - }; - }; - - const downloadSkippedRecords = () => { - const element = document.createElement("a"); - console.log(skippedRecords); - const file = new Blob([skippedRecords], { - type: "text/comma-separated-values", - }); - element.href = URL.createObjectURL(file); - element.download = "skippedRecords.csv"; - document.body.appendChild(element); // Required for this to work in FireFox - element.click(); - }; - - const onConflictModeChanged = async (e: ChangeEvent) => { - if (progress !== null) { - return; - } - - const value = e.target.value; - setConflictMode(value); - }; - - const onPasswordModeChange = (e: ChangeEvent) => { - if (progress !== null) { - return; - } - - setPasswordMode(e.target.checked); - }; - - const onUseridModeChanged = async (e: ChangeEvent) => { - if (progress !== null) { - return; - } - - const value = e.target.value; - setUseridMode(value); - }; - - const onDryRunModeChanged = (e: ChangeEvent) => { - if (progress !== null) { - return; - } - setDryRun(e.target.checked); - }; - - // render individual small components - - const statsCards = stats && - !importResults && [ - - - -
{translate("import_users.cards.importstats.users_total", stats.total)}
-
{translate("import_users.cards.importstats.guest_count", stats.is_guest)}
-
{translate("import_users.cards.importstats.admin_count", stats.admin)}
- - , - - - -
- {stats.id === stats.total - ? translate("import_users.cards.ids.all_ids_present") - : translate("import_users.cards.ids.count_ids_present", stats.id)} -
- {stats.id > 0 ? ( -
- - - - -
- ) : ( - "" - )} - - , - - - -
- {stats.password === stats.total - ? translate("import_users.cards.passwords.all_passwords_present") - : translate("import_users.cards.passwords.count_passwords_present", stats.password)} -
- {stats.password > 0 ? ( -
- - } - label={translate("import_users.cards.passwords.use_passwords")} - /> -
- ) : ( - "" - )} - - , - ]; - - const conflictCards = stats && !importResults && ( - - - -
- - - - -
- - - ); - - const errorCards = error && ( - - - - {(Array.isArray(error) ? error : [error]).map(e => ( -
{e}
- ))} - - - ); - - const uploadCard = !importResults && ( - - - - {translate("import_users.cards.upload.explanation")} - example.csv -
-
- - - - ); - - const resultsCard = importResults && ( - - -
- {translate("import_users.cards.results.total", importResults.totalRecordCount)} -
- {translate("import_users.cards.results.successful", importResults.succeededRecords.length)} -
- {importResults.skippedRecords.length - ? [ - translate("import_users.cards.results.skipped", importResults.skippedRecords.length), -
- -
, -
, - ] - : ""} - {importResults.erroredRecords.length - ? [translate("import_users.cards.results.skipped", importResults.erroredRecords.length),
] - : ""} -
- {importResults.wasDryRun && [translate("import_users.cards.results.simulated_only"),
]} -
- - ); - - const startImportCard = - !values || values.length === 0 || importResults ? undefined : ( - - } - label={translate("import_users.cards.startImport.simulate_only")} - /> - - {progress !== null ? ( -
- {progress.done} of {progress.limit} done -
- ) : null} - - ); - - const allCards: React.JSX.Element[] = []; - if (uploadCard) allCards.push(uploadCard); - if (errorCards) allCards.push(errorCards); - if (conflictCards) allCards.push(conflictCards); - if (statsCards) allCards.push(...statsCards); - if (startImportCard) allCards.push(startImportCard); - if (resultsCard) allCards.push(resultsCard); - - const cardContainer = {allCards}; - - return [, cardContainer]; -}; - -export const UserImport = FilePicker; -export default UserImport; diff --git a/src/components/user-import/ConflictModeCard.tsx b/src/components/user-import/ConflictModeCard.tsx new file mode 100644 index 0000000..b5c5d31 --- /dev/null +++ b/src/components/user-import/ConflictModeCard.tsx @@ -0,0 +1,40 @@ +import { NativeSelect, Paper } from "@mui/material"; + +import { CardContent, CardHeader, Container } from "@mui/material"; + +import { useTranslate } from "ra-core"; +import { ParsedStats, Progress } from "./types"; +import { ChangeEventHandler } from "react"; + +const TranslatableOption = ({ value, text }: { value: string, text: string }) => { + const translate = useTranslate(); + return <option value={value}>{translate(text)}</option>; +} + +const ConflictModeCard = ({ stats, importResults, onConflictModeChanged, conflictMode, progress }: + { stats: ParsedStats | null, importResults: any, onConflictModeChanged: ChangeEventHandler<HTMLSelectElement>, conflictMode: string, progress: Progress }) => { + const translate = useTranslate(); + + if (!stats || importResults) { + return null; + } + + return ( + <Container sx={{ mb: 3 }}> + <Paper elevation={1}> + <CardHeader + title={translate("import_users.cards.conflicts.header")} + sx={{ borderBottom: 1, borderColor: "divider" }} + /> + <CardContent> + <NativeSelect onChange={onConflictModeChanged} value={conflictMode} disabled={progress !== null}> + <TranslatableOption value="stop" text="import_users.cards.conflicts.mode.stop" /> + <TranslatableOption value="skip" text="import_users.cards.conflicts.mode.skip" /> + </NativeSelect> + </CardContent> + </Paper> + </Container> + ); +} + +export default ConflictModeCard; \ No newline at end of file diff --git a/src/components/user-import/ErrorsCard.tsx b/src/components/user-import/ErrorsCard.tsx new file mode 100644 index 0000000..ea4a29a --- /dev/null +++ b/src/components/user-import/ErrorsCard.tsx @@ -0,0 +1,43 @@ +import { + Container, + Paper, + CardHeader, + CardContent, + Stack, + Typography, +} from "@mui/material"; +import { useTranslate } from "ra-core"; + +const ErrorsCard = ({ errors }: { errors: string[] }) => { + const translate = useTranslate(); + + if (errors.length === 0) { + return null; + } + + return ( + <Container sx={{ mb: 3 }}> + <Paper elevation={1}> + <CardHeader + title={translate("import_users.error.error")} + sx={{ + borderBottom: 1, + borderColor: "error.main", + color: "error.main" + }} + /> + <CardContent> + <Stack spacing={1}> + {errors.map((e, idx) => ( + <Typography key={idx} color="error"> + {e} + </Typography> + ))} + </Stack> + </CardContent> + </Paper> + </Container> + ); +}; + +export default ErrorsCard; \ No newline at end of file diff --git a/src/components/user-import/ResultsCard.tsx b/src/components/user-import/ResultsCard.tsx new file mode 100644 index 0000000..467b735 --- /dev/null +++ b/src/components/user-import/ResultsCard.tsx @@ -0,0 +1,75 @@ +import { Alert, Box, CardContent, CardHeader, Container, List, ListItem, ListItemText, Paper, Stack, Typography } from "@mui/material" +import { Button, Link, useTranslate } from "react-admin"; +import { ImportResult } from "./types"; +import DownloadIcon from "@mui/icons-material/Download"; +import ArrowBackIcon from "@mui/icons-material/ArrowBack"; + +const ResultsCard = ({ importResults, downloadSkippedRecords }: { importResults: ImportResult | null, downloadSkippedRecords: () => void }) => { + const translate = useTranslate(); + + if (!importResults) { + return null; + } + + return ( + <Container> + <Paper> + <CardHeader + title={translate("import_users.cards.results.header")} + pb={0} + sx={{ + borderBottom: 1, + }} + /> + <CardContent> + <Stack spacing={2}> + <Typography key="total" color="text.primary"> + {translate("import_users.cards.results.total", importResults.totalRecordCount)} + </Typography> + <Typography key="successful" color="success.main"> + {translate("import_users.cards.results.successful", importResults.succeededRecords.length)} + </Typography> + <List dense> + {importResults.succeededRecords.map((record) => ( + <ListItem key={record.id}> + <ListItemText primary={record.displayname} /> + </ListItem> + ))} + </List> + {importResults.skippedRecords.length > 0 && ( + <Box> + <Typography key="skipped" color="warning.main"> + {translate("import_users.cards.results.skipped", importResults.skippedRecords.length)} + </Typography> + <Button + variant="outlined" + startIcon={<DownloadIcon />} + onClick={downloadSkippedRecords} + sx={{ mt: 2 }} + label={translate("import_users.cards.results.download_skipped")} + > + </Button> + </Box> + )} + {importResults.erroredRecords.length > 0 && ( + <Typography key="errored" color="error.main"> + {translate("import_users.cards.results.skipped", importResults.erroredRecords.length)} + </Typography> + )} + + {importResults.wasDryRun && ( + <Alert severity="warning" key="simulated"> + {translate("import_users.cards.results.simulated_only")} + </Alert> + )} + </Stack> + </CardContent> + </Paper> + <Box sx={{ mt: 2 }}> + <Link to="/users"><Button variant="outlined" startIcon={<ArrowBackIcon />} label={translate("ra.action.back")} /></Link> + </Box> + </Container> + ); +}; + +export default ResultsCard; diff --git a/src/components/user-import/StartImportCard.tsx b/src/components/user-import/StartImportCard.tsx new file mode 100644 index 0000000..781fb81 --- /dev/null +++ b/src/components/user-import/StartImportCard.tsx @@ -0,0 +1,39 @@ +import { Button, Checkbox, Paper, Container } from "@mui/material"; + +import { CardActions, FormControlLabel } from "@mui/material"; +import { Progress, ImportLine, ImportResult } from "./types"; +import { ChangeEventHandler } from "react"; +import { useTranslate } from "ra-core"; + +const StartImportCard = ( + { csvData, importResults, progress, dryRun, onDryRunModeChanged, runImport }: + { csvData: ImportLine[], importResults: ImportResult | null, progress: Progress, dryRun: boolean, onDryRunModeChanged: ChangeEventHandler<HTMLInputElement>, runImport: () => void } +) => { + const translate = useTranslate(); + if (!csvData || csvData.length === 0 || importResults) { + return null; + } + + return ( + <Container> + <Paper> + <CardActions> + <FormControlLabel + control={<Checkbox checked={dryRun} onChange={onDryRunModeChanged} disabled={progress !== null} />} + label={translate("import_users.cards.startImport.simulate_only")} + /> + <Button variant="contained" size="large" onClick={runImport} disabled={progress !== null}> + {translate("import_users.cards.startImport.run_import")} + </Button> + {progress !== null ? ( + <div> + {progress.done} of {progress.limit} done + </div> + ) : null} + </CardActions> + </Paper> + </Container> + ); +} + +export default StartImportCard; \ No newline at end of file diff --git a/src/components/user-import/StatsCard.tsx b/src/components/user-import/StatsCard.tsx new file mode 100644 index 0000000..77cadad --- /dev/null +++ b/src/components/user-import/StatsCard.tsx @@ -0,0 +1,83 @@ +import { Card, Paper, Stack, CardContent, CardHeader, Container, Typography } from "@mui/material"; +import { NativeSelect } from "@mui/material"; +import { FormControlLabel } from "@mui/material"; +import { Checkbox } from "@mui/material"; +import { useTranslate } from "ra-core"; +import { ChangeEventHandler } from "react"; +import { ParsedStats, Progress } from "./types"; + +const StatsCard = ({ stats, progress, importResults, useridMode, passwordMode, onUseridModeChanged, onPasswordModeChange }: { stats: ParsedStats | null, progress: Progress, importResults: any, useridMode: string, passwordMode: boolean, onUseridModeChanged: ChangeEventHandler<HTMLSelectElement>, onPasswordModeChange: ChangeEventHandler<HTMLInputElement> }) => { + const translate = useTranslate(); + + if (!stats) { + return null; + } + + if (importResults) { + return null; + } + + return ( + <> + <Container sx={{ mb: 3 }}> + <Paper> + <Card> + <CardHeader + title={translate("import_users.cards.importstats.header")} + sx={{ borderBottom: 1, borderColor: "divider" }} + /> + <CardContent> + <Stack spacing={1}> + <Typography>{translate("import_users.cards.importstats.users_total", stats.total)}</Typography> + <Typography>{translate("import_users.cards.importstats.guest_count", stats.is_guest)}</Typography> + <Typography>{translate("import_users.cards.importstats.admin_count", stats.admin)}</Typography> + </Stack> + </CardContent> + <CardHeader + title={translate("import_users.cards.ids.header")} + sx={{ borderBottom: 1, borderColor: "divider" }} + /> + <CardContent> + <Stack spacing={2}> + <Typography> + {stats.id === stats.total + ? translate("import_users.cards.ids.all_ids_present") + : translate("import_users.cards.ids.count_ids_present", stats.id)} + </Typography> + {stats.id > 0 && ( + <NativeSelect onChange={onUseridModeChanged} value={useridMode} disabled={progress !== null}> + <option value={"ignore"}>{translate("import_users.cards.ids.mode.ignore")}</option> + <option value={"update"}>{translate("import_users.cards.ids.mode.update")}</option> + </NativeSelect> + )} + </Stack> + </CardContent> + <CardHeader + title={translate("import_users.cards.passwords.header")} + sx={{ borderBottom: 1, borderColor: "divider" }} + /> + <CardContent> + <Stack spacing={1}> + <Typography> + {stats.password === stats.total + ? translate("import_users.cards.passwords.all_passwords_present") + : translate("import_users.cards.passwords.count_passwords_present", stats.password)} + </Typography> + {stats.password > 0 && ( + <FormControlLabel + control={ + <Checkbox checked={passwordMode} disabled={progress !== null} onChange={onPasswordModeChange} /> + } + label={translate("import_users.cards.passwords.use_passwords")} + /> + )} + </Stack> + </CardContent> + </Card> + </Paper> + </Container> + </> + ); +}; + +export default StatsCard; \ No newline at end of file diff --git a/src/components/user-import/UploadCard.tsx b/src/components/user-import/UploadCard.tsx new file mode 100644 index 0000000..75e4b70 --- /dev/null +++ b/src/components/user-import/UploadCard.tsx @@ -0,0 +1,36 @@ +import { CardHeader, CardContent, Container, Link, Stack, Typography, Paper } from "@mui/material"; + +import { useTranslate } from "ra-core"; +import { ChangeEventHandler } from "react"; +import { Progress } from "./types"; + +const UploadCard = ({ importResults, onFileChange, progress }: { importResults: any, onFileChange: ChangeEventHandler<HTMLInputElement>, progress: Progress }) => { + const translate = useTranslate(); + if (importResults) { + return null; + } + + return ( + <Container sx={{ mb: 3 }}> + <Paper elevation={1}> + <CardHeader + title={translate("import_users.cards.upload.header")} + sx={{ borderBottom: 1, borderColor: "divider" }} + /> + <CardContent> + <Stack spacing={2}> + <Typography> + {translate("import_users.cards.upload.explanation")} + <Link href="./data/example.csv" sx={{ ml: 1 }}> + example.csv + </Link> + </Typography> + <input type="file" onChange={onFileChange} disabled={progress !== null} /> + </Stack> + </CardContent> + </Paper> + </Container> + ); +}; + +export default UploadCard; \ No newline at end of file diff --git a/src/components/user-import/UserImport.tsx b/src/components/user-import/UserImport.tsx new file mode 100644 index 0000000..318a204 --- /dev/null +++ b/src/components/user-import/UserImport.tsx @@ -0,0 +1,51 @@ +import { + Stack, +} from "@mui/material"; +import { useTranslate } from "ra-core"; +import { Title } from "react-admin"; + +import UploadCard from "./UploadCard"; +import useImportFile from "./useImportFile"; +import ErrorsCard from "./ErrorsCard"; +import ConflictModeCard from "./ConflictModeCard"; +import StatsCard from "./StatsCard"; +import StartImportCard from "./StartImportCard"; +import ResultsCard from "./ResultsCard"; + +const UserImport = () => { + const { + csvData, + dryRun, + importResults, + progress, + errors, + stats, + conflictMode, + passwordMode, + useridMode, + onFileChange, + onDryRunModeChanged, + runImport, + onConflictModeChanged, + onPasswordModeChange, + onUseridModeChanged, + downloadSkippedRecords + } = useImportFile(); + + const translate = useTranslate(); + + return ( + <Stack spacing={3} mt={3} direction="column"> + <Title defaultTitle={translate("import_users.title")} /> + <UploadCard importResults={importResults} onFileChange={onFileChange} progress={progress} /> + <ErrorsCard errors={errors} /> + <ConflictModeCard stats={stats} importResults={importResults} conflictMode={conflictMode} onConflictModeChanged={onConflictModeChanged} progress={progress} /> + <StatsCard stats={stats} progress={progress} importResults={importResults} passwordMode={passwordMode} useridMode={useridMode} onPasswordModeChange={onPasswordModeChange} onUseridModeChanged={onUseridModeChanged} /> + <StartImportCard csvData={csvData} importResults={importResults} progress={progress} dryRun={dryRun} onDryRunModeChanged={onDryRunModeChanged} runImport={runImport} /> + <ResultsCard importResults={importResults} downloadSkippedRecords={downloadSkippedRecords} /> + </Stack> + ); + +}; + +export default UserImport; diff --git a/src/components/user-import/types.ts b/src/components/user-import/types.ts new file mode 100644 index 0000000..f790249 --- /dev/null +++ b/src/components/user-import/types.ts @@ -0,0 +1,47 @@ +import { RaRecord } from "react-admin"; + +export interface ImportLine { + id: string; + displayname: string; + user_type?: string; + name?: string; + deactivated?: boolean; + is_guest?: boolean; + admin?: boolean; + is_admin?: boolean; + password?: string; + avatar_url?: string; +} + +export interface ParsedStats { + user_types: { [key: string]: number }; + is_guest: number; + admin: number; + deactivated: number; + password: number; + avatar_url: number; + id: number; + total: number; +} + +export interface ChangeStats { + total: number; + id: number; + is_guest: number; + admin: number; + password: number; +} + +export type Progress = { + done: number; + limit: number; +} | null; + +export interface ImportResult { + skippedRecords: RaRecord[]; + erroredRecords: RaRecord[]; + succeededRecords: RaRecord[]; + totalRecordCount: number; + changeStats: ChangeStats; + wasDryRun: boolean; +} \ No newline at end of file diff --git a/src/components/user-import/useImportFile.tsx b/src/components/user-import/useImportFile.tsx new file mode 100644 index 0000000..49501de --- /dev/null +++ b/src/components/user-import/useImportFile.tsx @@ -0,0 +1,377 @@ +import { ChangeEvent, useState } from "react"; +import { useTranslate, useNotify, HttpError } from "react-admin"; +import { parse as parseCsv, unparse as unparseCsv, ParseResult } from "papaparse"; +import dataProvider from "../../synapse/dataProvider"; +import { returnMXID } from "../../utils/mxid"; +import { generateRandomPassword } from "../../utils/password"; +import { generateRandomMXID } from "../../utils/mxid"; +import { ImportLine, ParsedStats, Progress, ImportResult, ChangeStats } from "./types"; + +const LOGGING = true; + +const EXPECTED_FIELDS = ["id", "displayname"].sort(); + +const useImportFile = () => { + const [csvData, setCsvData] = useState<ImportLine[]>([]); + const [errors, setErrors] = useState<string[]>([]); + const [stats, setStats] = useState<ParsedStats | null>(null); + const [dryRun, setDryRun] = useState(true); + + const [progress, setProgress] = useState<Progress>(null); + + const [importResults, setImportResults] = useState<ImportResult | null>(null); + const [skippedRecords, setSkippedRecords] = useState<string>(""); + + const [conflictMode, setConflictMode] = useState<"stop" | "skip">("stop"); + const [passwordMode, setPasswordMode] = useState(true); + const [useridMode, setUseridMode] = useState<"update" | "ignore">("update"); + + const translate = useTranslate(); + const notify = useNotify(); + + const onFileChange = async (e: ChangeEvent<HTMLInputElement>) => { + if (progress !== null) return; + + setCsvData([]); + setErrors([]); + setStats(null); + setImportResults(null); + const file = e.target.files ? e.target.files[0] : null; + if (!file) return; + /* Let's refuse some unreasonably big files instead of freezing + * up the browser */ + if (file.size > 100000000) { + const message = translate("import_users.errors.unreasonably_big", { + size: (file.size / (1024 * 1024)).toFixed(2), + }); + notify(message); + setErrors([message]); + return; + } + try { + parseCsv<ImportLine>(file, { + header: true, + skipEmptyLines: true /* especially for a final EOL in the csv file */, + complete: result => { + if (result.errors) { + setErrors(result.errors.map(e => e.toString())); + } + /* Papaparse is very lenient, we may be able to salvage + * the data in the file. */ + verifyCsv(result); + }, + }); + } catch { + setErrors(["Unknown error"]); + return null; + } + }; + + const verifyCsv = ({ data, meta, errors }: ParseResult<ImportLine>) => { + /* First, verify the presence of required fields */ + meta.fields = meta.fields?.map(f => f.trim().toLowerCase()); + const missingFields = EXPECTED_FIELDS.filter(eF => !meta.fields?.find(mF => eF === mF)); + + if (missingFields.length > 0) { + setErrors([translate("import_users.error.required_field", { field: missingFields[0] })]); + return false; + } + + + /* Collect some stats to prevent sneaky csv files from adding admin + users or something. + */ + const stats: ParsedStats = { + user_types: { default: 0 }, + is_guest: 0, + admin: 0, + deactivated: 0, + password: 0, + avatar_url: 0, + id: 0, + + total: data.length, + }; + + const errorMessages = errors.map(e => e.message); + // sanitize the data first + data = data.map(line => { + const newLine = {} as ImportLine; + for (const [key, value] of Object.entries(line)) { + newLine[key.trim().toLowerCase()] = value; + } + return newLine; + }); + + // process the data + data.forEach((line, idx) => { + if (line.user_type === undefined || line.user_type === "") { + stats.user_types.default++; + } else { + stats.user_types[line.user_type] += 1; + } + /* XXX correct the csv export that react-admin offers for the users + * resource so it gives sensible field names and doesn't duplicate + * id as "name"? + */ + if (meta.fields?.includes("name")) { + delete line.name; + } + if (meta.fields?.includes("user_type")) { + delete line.user_type; + } + if (meta.fields?.includes("is_admin")) { + delete line.is_admin; + } + + ["is_guest", "admin", "deactivated"].forEach(f => { + if (line[f] === "true") { + stats[f]++; + line[f] = true; // we need true booleans instead of strings + } else { + if (line[f] !== "false" && line[f] !== "") { + console.log("invalid value", line[f], "for field " + f + " in row " + idx); + errorMessages.push( + translate("import_users.error.invalid_value", { + field: f, + row: idx, + }) + ); + } + line[f] = false; // default values to false + } + }); + + if (line.password !== undefined && line.password !== "") { + stats.password++; + } + + if (line.avatar_url !== undefined && line.avatar_url !== "") { + stats.avatar_url++; + } + + if (line.id !== undefined && line.id !== "") { + stats.id++; + } + }); + + if (errorMessages.length > 0) { + setErrors(errorMessages); + return false; + } + + setStats(stats); + setCsvData(data); + + return true; + }; + + const onConflictModeChanged = async (e: ChangeEvent<HTMLSelectElement>) => { + if (progress !== null) { + return; + } + + const value = e.target.value as "stop" | "skip"; + setConflictMode(value); + }; + + const onPasswordModeChange = (e: ChangeEvent<HTMLInputElement>) => { + if (progress !== null) { + return; + } + + setPasswordMode(e.target.checked); + }; + + const onUseridModeChanged = (e: ChangeEvent<HTMLSelectElement>) => { + if (progress !== null) { + return; + } + + const value = e.target.value as "update" | "ignore"; + setUseridMode(value); + }; + + const onDryRunModeChanged = (e: ChangeEvent<HTMLInputElement>) => { + if (progress !== null) { + return; + } + setDryRun(e.target.checked); + }; + + const runImport = async () => { + if (progress !== null) { + notify("import_users.errors.already_in_progress"); + return; + } + + const results = await doImport(); + setImportResults(results); + // offer CSV download of skipped or errored records + // (so that the user doesn't have to filter out successful + // records manually when fixing stuff in the CSV) + setSkippedRecords(unparseCsv(results.skippedRecords)); + if (LOGGING) console.log("Skipped records:"); + if (LOGGING) console.log(skippedRecords); + }; + + const doImport = async (): Promise<ImportResult> => { + const skippedRecords: ImportLine[] = []; + const erroredRecords: ImportLine[] = []; + const succeededRecords: ImportLine[] = []; + const changeStats: ChangeStats = { + total: 0, + id: 0, + is_guest: 0, + admin: 0, + password: 0, + }; + let entriesDone = 0; + const entriesCount = csvData.length; + try { + setProgress({ done: entriesDone, limit: entriesCount }); + for (const entry of csvData) { + const userRecord = { ...entry }; + // No need to do a bunch of cryptographic random number getting if + // we are using neither a generated password nor a generated user id. + if (useridMode === "ignore" || userRecord.id === undefined || userRecord.id === "") { + userRecord.id = generateRandomMXID(); + } + if (passwordMode === false || entry.password === undefined || entry.password === "") { + userRecord.password = generateRandomPassword(); + } + // we want to ensure that the ID is always full MXID, otherwise randomly-generated MXIDs will be in the full + // form, but the ones from the CSV will be localpart-only. + userRecord.id = returnMXID(userRecord.id); + /* TODO record update stats (especially admin no -> yes, deactivated x -> !x, ... */ + + /* For these modes we will consider the ID that's in the record. + * If the mode is "stop", we will not continue adding more records, and + * we will offer information on what was already added and what was + * skipped. + * + * If the mode is "skip", we record the record for later, but don't + * send it to the server. + * + * If the mode is "update", we change fields that are reasonable to + * update. + * - If the "password mode" is "true" (i.e. "use passwords from csv"): + * - if the record has a password + * - send the password along with the record + * - if the record has no password + * - generate a new password + * - If the "password mode" is "false" + * - never generate a new password to update existing users with + */ + + /* We just act as if there are no IDs in the CSV, so every user will be + * created anew. + * We do a simple retry loop so that an accidental hit on an existing ID + * doesn't trip us up. + */ + if (LOGGING) console.log("will check for existence of record " + JSON.stringify(userRecord)); + let retries = 0; + const submitRecord = async (recordData: ImportLine) => { + try { + const response = await dataProvider.getOne("users", { id: recordData.id }); + + if (LOGGING) console.log("already existed"); + + if (conflictMode === "stop") { + throw new Error( + translate("import_users.error.id_exits", { + id: recordData.id, + }) + ); + } + + if (conflictMode === "skip" || useridMode === "update") { + skippedRecords.push(recordData); + return; + } + + const newRecordData = Object.assign({}, recordData, { + id: generateRandomMXID(), + }); + retries++; + + if (retries > 512) { + console.warn("retry loop got stuck? pathological situation?"); + skippedRecords.push(recordData); + return; + } + + await submitRecord(newRecordData); + } catch (e) { + if (!(e instanceof HttpError) || (e.status && e.status !== 404)) { + throw e; + } + + if (LOGGING) console.log("OK to create record " + recordData.id + " (" + recordData.displayname + ")."); + + if (!dryRun) { + await dataProvider.create("users", { data: recordData }); + } + succeededRecords.push(recordData); + } + }; + + await submitRecord(userRecord); + entriesDone++; + setProgress({ done: entriesDone, limit: csvData.length }); + } + + setProgress(null); + } catch (e) { + setErrors([ + translate("import_users.error.at_entry", { + entry: entriesDone + 1, + message: e instanceof Error ? e.message : String(e), + }) + ]); + setProgress(null); + } + + return { + skippedRecords, + erroredRecords, + succeededRecords, + totalRecordCount: entriesCount, + changeStats, + wasDryRun: dryRun, + }; + }; + + const downloadSkippedRecords = () => { + const element = document.createElement("a"); + console.log(skippedRecords); + const file = new Blob([skippedRecords], { + type: "text/comma-separated-values", + }); + element.href = URL.createObjectURL(file); + element.download = "skippedRecords.csv"; + document.body.appendChild(element); // Required for this to work in FireFox + element.click(); + }; + + return { + csvData, + dryRun, + onDryRunModeChanged, + runImport, + progress, + importResults, + errors, + stats, + conflictMode, + passwordMode, + useridMode, + onConflictModeChanged, + onPasswordModeChange, + onUseridModeChanged, + onFileChange, + downloadSkippedRecords + } +} + +export default useImportFile; \ No newline at end of file diff --git a/src/i18n/de.ts b/src/i18n/de.ts index 3888870..0323c93 100644 --- a/src/i18n/de.ts +++ b/src/i18n/de.ts @@ -91,7 +91,7 @@ const de: SynapseTranslationMessages = { goToPdf: "Gehe zum PDF", cards: { importstats: { - header: "Benutzer importieren", + header: "Geparste Benutzer für den Import", users_total: "%{smart_count} Benutzer in der CSV Datei |||| %{smart_count} Benutzer in der CSV Datei", guest_count: "%{smart_count} Gast |||| %{smart_count} Gäste", admin_count: "%{smart_count} Server Administrator |||| %{smart_count} Server Administratoren", diff --git a/src/i18n/en.ts b/src/i18n/en.ts index de50026..7efdf91 100644 --- a/src/i18n/en.ts +++ b/src/i18n/en.ts @@ -58,7 +58,7 @@ const en: SynapseTranslationMessages = { goToPdf: "Go to PDF", cards: { importstats: { - header: "Import users", + header: "Parsed users for import", users_total: "%{smart_count} user in CSV file |||| %{smart_count} users in CSV file", guest_count: "%{smart_count} guest |||| %{smart_count} guests", admin_count: "%{smart_count} admin |||| %{smart_count} admins", diff --git a/src/i18n/fa.ts b/src/i18n/fa.ts index c7f6a7f..c864890 100644 --- a/src/i18n/fa.ts +++ b/src/i18n/fa.ts @@ -51,7 +51,7 @@ const fa: SynapseTranslationMessages = { goToPdf: "رفتن به PDF", cards: { importstats: { - header: "وارد کردن کاربران", + header: "کاربران پردازش شده برای وارد کردن", users_total: "%{smart_count} user in CSV file |||| %{smart_count} users in CSV file", guest_count: "%{smart_count} guest |||| %{smart_count} guests", admin_count: "%{smart_count} admin |||| %{smart_count} admins", diff --git a/src/i18n/fr.ts b/src/i18n/fr.ts index 2158d87..3f45cb6 100644 --- a/src/i18n/fr.ts +++ b/src/i18n/fr.ts @@ -52,7 +52,7 @@ const fr: SynapseTranslationMessages = { goToPdf: "Voir le PDF", cards: { importstats: { - header: "Importer des utilisateurs", + header: "Utilisateurs analysés pour l'import", users_total: "%{smart_count} utilisateur dans le fichier CSV |||| %{smart_count} utilisateurs dans le fichier CSV", guest_count: "%{smart_count} visiteur |||| %{smart_count} visiteurs", diff --git a/src/i18n/it.ts b/src/i18n/it.ts index 204d26a..3414b3e 100644 --- a/src/i18n/it.ts +++ b/src/i18n/it.ts @@ -51,7 +51,7 @@ const it: SynapseTranslationMessages = { goToPdf: "Vai al PDF", cards: { importstats: { - header: "Importa utenti", + header: "Utenti analizzati per l'importazione", users_total: "%{smart_count} utente nel file CSV |||| %{smart_count} utenti nel file CSV", guest_count: "%{smart_count} ospite |||| %{smart_count} ospiti", admin_count: "%{smart_count} amministratore |||| %{smart_count} amministratori", diff --git a/src/i18n/ru.ts b/src/i18n/ru.ts index 1a29492..3f4950d 100644 --- a/src/i18n/ru.ts +++ b/src/i18n/ru.ts @@ -83,7 +83,7 @@ const ru: SynapseTranslationMessages = { goToPdf: "Перейти к PDF", cards: { importstats: { - header: "Импорт пользователей", + header: "Анализированные пользователи для импорта", users_total: "%{smart_count} пользователь в CSV файле |||| %{smart_count} пользователя в CSV файле |||| %{smart_count} пользователей в CSV файле", guest_count: "%{smart_count} гость |||| %{smart_count} гостя |||| %{smart_count} гостей", diff --git a/src/i18n/zh.ts b/src/i18n/zh.ts index 9fe16cf..46c22e4 100644 --- a/src/i18n/zh.ts +++ b/src/i18n/zh.ts @@ -86,7 +86,7 @@ const zh: SynapseTranslationMessages = { goToPdf: "转到 PDF", cards: { importstats: { - header: "导入用户", + header: "分析用于导入的用户", users_total: "%{smart_count} 用户在 CSV 文件中 |||| %{smart_count} 用户在 CSV 文件中", guest_count: "%{smart_count} 访客 |||| %{smart_count} 访客", admin_count: "%{smart_count} 管理员 |||| %{smart_count} 管理员",