Import users from CSV

Change-Id: I3565dae7531b7de1d68ebcb2392be931083dda9e
This commit is contained in:
Michael Albert
2020-09-22 11:30:27 +02:00
parent 3d4f45d070
commit e5d2d895d9
8 changed files with 946 additions and 21 deletions

View File

@@ -0,0 +1,674 @@
import React, { useState } from "react";
import {
Button as ReactAdminButton,
useDataProvider,
useNotify,
Title,
} from "react-admin";
import { parse as parseCsv, unparse as unparseCsv } from "papaparse";
import GetAppIcon from "@material-ui/icons/GetApp";
import {
Button,
Card,
CardActions,
CardContent,
CardHeader,
FormControlLabel,
Checkbox,
NativeSelect,
} from "@material-ui/core";
import { useTranslate } from "ra-core";
import Container from "@material-ui/core/Container/Container";
import { generateRandomUser } from "./users";
const LOGGING = true;
export const ImportButton = ({ label, variant = "text" }) => {
return (
<ReactAdminButton
color="primary"
component="span"
variant={variant}
label={label}
>
<GetAppIcon style={{ transform: "rotate(180deg)", fontSize: "20" }} />
</ReactAdminButton>
);
};
const expectedFields = ["id", "displayname"].sort();
const optionalFields = [
"user_type",
"guest",
"admin",
"deactivated",
"avatar_url",
"password",
].sort();
function TranslatableOption({ value, text }) {
const translate = useTranslate();
return <option value={value}>{translate(text)}</option>;
}
const FilePicker = props => {
const [values, setValues] = useState(null);
const [error, setError] = useState(null);
const [stats, setStats] = useState(null);
const [dryRun, setDryRun] = useState(true);
const [progress, setProgress] = useState(null);
const [importResults, setImportResults] = useState(null);
const [skippedRecords, setSkippedRecords] = useState(null);
const [conflictMode, setConflictMode] = useState("stop");
const [passwordMode, setPasswordMode] = useState(true);
const [useridMode, setUseridMode] = useState("ignore");
const translate = useTranslate();
const notify = useNotify();
const dataProvider = useDataProvider();
const onFileChange = async e => {
if (progress !== null) return;
setValues(null);
setError(null);
setStats(null);
setImportResults(null);
const file = e.target.files ? e.target.files[0] : null;
/* Let's refuse some unreasonably big files instead of freezing
* up the browser */
if (file.size > 100000000) {
const message = translate("import_users.errors.unreasonably_big", {
size: (file.size / (1024 * 1024)).toFixed(2),
});
notify(message);
setError(message);
return;
}
try {
parseCsv(file, {
header: true,
skipEmptyLines: true /* especially for a final EOL in the csv file */,
complete: result => {
if (result.error) {
setError(result.error);
}
/* Papaparse is very lenient, we may be able to salvage
* the data in the file. */
verifyCsv(result, { setValues, setStats, setError });
},
});
} catch {
setError(true);
return null;
}
};
const verifyCsv = (
{ data, meta, errors },
{ setValues, setStats, setError }
) => {
/* First, verify the presence of required fields */
let eF = Array.from(expectedFields);
let oF = Array.from(optionalFields);
meta.fields.forEach(name => {
if (eF.includes(name)) {
eF = eF.filter(v => v !== name);
}
if (oF.includes(name)) {
oF = oF.filter(v => v !== name);
}
});
if (eF.length !== 0) {
setError(
translate("import_users.error.required_field", { field: eF[0] })
);
return false;
}
// XXX after deciding on how "name" and friends should be handled below,
// this place will want changes, too.
/* Collect some stats to prevent sneaky csv files from adding admin
users or something.
*/
let stats = {
user_types: { default: 0 },
is_guest: 0,
admin: 0,
deactivated: 0,
password: 0,
avatar_url: 0,
id: 0,
total: data.length,
};
data.forEach((line, idx) => {
if (line.user_type === undefined || line.user_type === "") {
stats.user_types.default++;
} else {
stats.user_types[line.user_type] += 1;
}
/* XXX correct the csv export that react-admin offers for the users
* resource so it gives sensible field names and doesn't duplicate
* id as "name"?
*/
if (meta.fields.includes("name")) {
delete line.name;
}
if (meta.fields.includes("user_type")) {
delete line.user_type;
}
if (meta.fields.includes("is_admin")) {
line.admin = line.is_admin;
delete line.is_admin;
}
["is_guest", "admin", "deactivated"].forEach(f => {
if (line[f] === "true") {
stats[f]++;
line[f] = true; // we need true booleans instead of strings
} else {
if (line[f] !== "false" && line[f] !== "") {
errors.push(
translate("import_users.error.invalid_value", {
field: f,
row: idx,
})
);
}
line[f] = false; // default values to false
}
});
if (line.password !== undefined && line.password !== "") {
stats.password++;
}
if (line.avatar_url !== undefined && line.avatar_url !== "") {
stats.avatar_url++;
}
if (line.id !== undefined && line.id !== "") {
stats.id++;
}
});
if (errors.length > 0) {
setError(errors);
}
setStats(stats);
setValues(data);
return true;
};
const runImport = async e => {
if (progress !== null) {
notify("import_users.errors.already_in_progress");
return;
}
const results = await doImport(
dataProvider,
values,
conflictMode,
passwordMode,
useridMode,
dryRun,
setProgress,
setError
);
setImportResults(results);
// offer CSV download of skipped or errored records
// (so that the user doesn't have to filter out successful
// records manually when fixing stuff in the CSV)
setSkippedRecords(unparseCsv(results.skippedRecords));
if (LOGGING) console.log("Skipped records:");
if (LOGGING) console.log(skippedRecords);
};
// XXX every single one of the requests will restart the activity indicator
// which doesn't look very good.
const doImport = async (
dataProvider,
data,
conflictMode,
passwordMode,
useridMode,
dryRun,
setProgress,
setError
) => {
let skippedRecords = [];
let erroredRecords = [];
let succeededRecords = [];
let changeStats = {
toAdmin: 0,
toGuest: 0,
toRegular: 0,
replacedPassword: 0,
};
let entriesDone = 0;
let entriesCount = data.length;
try {
setProgress({ done: entriesDone, limit: entriesCount });
for (const entry of data) {
let userRecord = {};
let overwriteData = {};
// No need to do a bunch of cryptographic random number getting if
// we are using neither a generated password nor a generated user id.
if (
useridMode === "ignore" ||
entry.id === undefined ||
entry.password === undefined ||
passwordMode === false
) {
overwriteData = generateRandomUser();
// Ignoring IDs or the entry lacking an ID means we keep the
// ID field in the overwrite data.
if (!(useridMode === "ignore" || entry.id === undefined)) {
delete overwriteData.id;
}
// Not using passwords from the csv or this entry lacking a password
// means we keep the password field in the overwrite data.
if (
!(
passwordMode === false ||
entry.password === undefined ||
entry.password === ""
)
) {
delete overwriteData.password;
}
}
/* TODO record update stats (especially admin no -> yes, deactivated x -> !x, ... */
Object.assign(userRecord, entry);
Object.assign(userRecord, overwriteData);
/* For these modes we will consider the ID that's in the record.
* If the mode is "stop", we will not continue adding more records, and
* we will offer information on what was already added and what was
* skipped.
*
* If the mode is "skip", we record the record for later, but don't
* send it to the server.
*
* If the mode is "update", we change fields that are reasonable to
* update.
* - If the "password mode" is "true" (i.e. "use passwords from csv"):
* - if the record has a password
* - send the password along with the record
* - if the record has no password
* - generate a new password
* - If the "password mode" is "false"
* - never generate a new password to update existing users with
*/
/* We just act as if there are no IDs in the CSV, so every user will be
* created anew.
* We do a simple retry loop so that an accidental hit on an existing ID
* doesn't trip us up.
*/
if (LOGGING)
console.log(
"will check for existence of record " + JSON.stringify(userRecord)
);
let retries = 0;
const submitRecord = recordData => {
return dataProvider.getOne("users", { id: recordData.id }).then(
async alreadyExists => {
if (LOGGING) console.log("already existed");
if (useridMode === "update" || conflictMode === "skip") {
skippedRecords.push(recordData);
} else if (conflictMode === "stop") {
throw new Error(
translate("import_users.error.id_exits", {
id: recordData.id,
})
);
} else {
const overwriteData = generateRandomUser();
const newRecordData = Object.assign({}, recordData, {
id: overwriteData.id,
});
retries++;
if (retries > 512) {
console.warn("retry loop got stuck? pathological situation?");
skippedRecords.push(recordData);
} else {
await submitRecord(newRecordData);
}
}
},
async okToSubmit => {
if (LOGGING)
console.log(
"OK to create record " +
recordData.id +
" (" +
recordData.displayname +
")."
);
if (!dryRun) {
await dataProvider.create("users", { data: recordData });
}
succeededRecords.push(recordData);
}
);
};
await submitRecord(userRecord);
entriesDone++;
setProgress({ done: entriesDone, limit: data.length });
}
setProgress(null);
} catch (e) {
setError(
translate("import_users.error.at_entry", {
entry: entriesDone + 1,
message: e.message,
})
);
setProgress(null);
}
return {
skippedRecords,
erroredRecords,
succeededRecords,
totalRecordCount: entriesCount,
changeStats,
wasDryRun: dryRun,
};
};
const downloadSkippedRecords = () => {
const element = document.createElement("a");
console.log(skippedRecords);
const file = new Blob([skippedRecords], {
type: "text/comma-separated-values",
});
element.href = URL.createObjectURL(file);
element.download = "skippedRecords.csv";
document.body.appendChild(element); // Required for this to work in FireFox
element.click();
};
const onConflictModeChanged = async e => {
if (progress !== null) {
return;
}
const value = e.target.value;
setConflictMode(value);
};
const onPasswordModeChange = e => {
if (progress !== null) {
return;
}
setPasswordMode(e.target.checked);
};
const onUseridModeChanged = async e => {
if (progress !== null) {
return;
}
const value = e.target.value;
setUseridMode(value);
};
const onDryRunModeChanged = ev => {
if (progress !== null) {
return;
}
setDryRun(ev.target.checked);
};
// render individual small components
const statsCards = stats &&
!importResults && [
<Container>
<CardHeader
title={translate("import_users.cards.importstats.header")}
/>
<CardContent>
<div>
{translate(
"import_users.cards.importstats.users_total",
stats.total
)}
</div>
<div>
{translate(
"import_users.cards.importstats.guest_count",
stats.is_guest
)}
</div>
<div>
{translate(
"import_users.cards.importstats.admin_count",
stats.admin
)}
</div>
</CardContent>
</Container>,
<Container>
<CardHeader title={translate("import_users.cards.ids.header")} />
<CardContent>
<div>
{stats.id === stats.total
? translate("import_users.cards.ids.all_ids_present")
: translate("import_users.cards.ids.count_ids_present", stats.id)}
</div>
{stats.id > 0 ? (
<div>
<NativeSelect
onChange={onUseridModeChanged}
value={useridMode}
enabled={(progress !== null).toString()}
>
<TranslatableOption
value="ignore"
text="import_users.cards.ids.mode.ignore"
/>
<TranslatableOption
value="update"
text="import_users.cards.ids.mode.update"
/>
</NativeSelect>
</div>
) : (
""
)}
</CardContent>
</Container>,
<Container>
<CardHeader title={translate("import_users.cards.passwords.header")} />
<CardContent>
<div>
{stats.password === stats.total
? translate("import_users.cards.passwords.all_passwords_present")
: translate(
"import_users.cards.passwords.count_passwords_present",
stats.password
)}
</div>
{stats.password > 0 ? (
<div>
<FormControlLabel
control={
<Checkbox
checked={passwordMode}
enabled={(progress !== null).toString()}
onChange={onPasswordModeChange}
/>
}
label={translate("import_users.cards.passwords.use_passwords")}
/>
</div>
) : (
""
)}
</CardContent>
</Container>,
];
let conflictCards = stats && !importResults && (
<Container>
<CardHeader title={translate("import_users.cards.conflicts.header")} />
<CardContent>
<div>
<NativeSelect
onChange={onConflictModeChanged}
value={conflictMode}
enabled={(progress !== null).toString()}
>
<TranslatableOption
value="stop"
text="import_users.cards.conflicts.mode.stop"
/>
<TranslatableOption
value="skip"
text="import_users.cards.conflicts.mode.skip"
/>
</NativeSelect>
</div>
</CardContent>
</Container>
);
let errorCards = error && (
<Container>
<CardHeader title={translate("import_users.error.error")} />
<CardContent>
{(Array.isArray(error) ? error : [error]).map(e => (
<div>{e}</div>
))}
</CardContent>
</Container>
);
let uploadCard = !importResults && (
<Container>
<CardHeader title={translate("import_users.cards.upload.header")} />
<CardContent>
{translate("import_users.cards.upload.explanation")}
<a href="/data/example.csv">example.csv</a>
<br />
<br />
<input
type="file"
onChange={onFileChange}
enabled={(progress !== null).toString()}
/>
</CardContent>
</Container>
);
let resultsCard = importResults && (
<CardContent>
<CardHeader title={translate("import_users.cards.results.header")} />
<div>
{translate(
"import_users.cards.results.total",
importResults.totalRecordCount
)}
<br />
{translate(
"import_users.cards.results.successful",
importResults.succeededRecords.length
)}
<br />
{importResults.skippedRecords.length
? [
translate(
"import_users.cards.results.skipped",
importResults.skippedRecords.length
),
<div>
<button onClick={downloadSkippedRecords}>
{translate("import_users.cards.results.download_skipped")}
</button>
</div>,
<br />,
]
: ""}
{importResults.erroredRecords.length
? [
translate(
"import_users.cards.results.skipped",
importResults.erroredRecords.length
),
<br />,
]
: ""}
<br />
{importResults.wasDryRun && [
translate("import_users.cards.results.simulated_only"),
<br />,
]}
</div>
</CardContent>
);
let startImportCard =
!values || values.length === 0 || importResults ? undefined : (
<CardActions>
<FormControlLabel
control={
<Checkbox
checked={dryRun}
onChange={onDryRunModeChanged}
enabled={(progress !== null).toString()}
/>
}
label={translate("import_users.cards.startImport.simulate_only")}
/>
<Button
size="large"
onClick={runImport}
enabled={(progress !== null).toString()}
>
{translate("import_users.cards.startImport.run_import")}
</Button>
{progress !== null ? (
<div>
{progress.done} of {progress.limit} done
</div>
) : null}
</CardActions>
);
let allCards = [];
if (uploadCard) allCards.push(uploadCard);
if (errorCards) allCards.push(errorCards);
if (conflictCards) allCards.push(conflictCards);
if (statsCards) allCards.push(...statsCards);
if (startImportCard) allCards.push(startImportCard);
if (resultsCard) allCards.push(resultsCard);
let cardContainer = <Card>{allCards}</Card>;
return [
<Title defaultTitle={translate("import_users.title")} />,
cardContainer,
];
};
export const ImportFeature = FilePicker;

39
src/components/Menu.js Normal file
View File

@@ -0,0 +1,39 @@
// in src/Menu.js
import * as React from "react";
import { useSelector } from "react-redux";
import { useMediaQuery } from "@material-ui/core";
import { MenuItemLink, getResources } from "react-admin";
import DefaultIcon from "@material-ui/icons/ViewList";
import LabelIcon from "@material-ui/icons/Label";
const Menu = ({ onMenuClick, logout }) => {
const isXSmall = useMediaQuery(theme => theme.breakpoints.down("xs"));
const open = useSelector(state => state.admin.ui.sidebarOpen);
const resources = useSelector(getResources);
return (
<div>
{resources.map(resource => (
<MenuItemLink
key={resource.name}
to={`/${resource.name}`}
primaryText={
(resource.options && resource.options.label) || resource.name
}
leftIcon={resource.icon ? <resource.icon /> : <DefaultIcon />}
onClick={onMenuClick}
sidebarIsOpen={open}
/>
))}
<MenuItemLink
to="/custom-route"
primaryText="Miscellaneous"
leftIcon={<LabelIcon />}
onClick={onMenuClick}
sidebarIsOpen={open}
/>
{isXSmall && logout}
</div>
);
};
export default Menu;

View File

@@ -3,10 +3,12 @@ import Avatar from "@material-ui/core/Avatar";
import PersonPinIcon from "@material-ui/icons/PersonPin";
import ContactMailIcon from "@material-ui/icons/ContactMail";
import DevicesIcon from "@material-ui/icons/Devices";
import GetAppIcon from "@material-ui/icons/GetApp";
import SettingsInputComponentIcon from "@material-ui/icons/SettingsInputComponent";
import {
ArrayInput,
ArrayField,
Button,
Datagrid,
DateField,
Create,
@@ -31,6 +33,7 @@ import {
DeleteButton,
SaveButton,
regex,
useRedirect,
useTranslate,
Pagination,
CreateButton,
@@ -42,6 +45,12 @@ import { ServerNoticeButton, ServerNoticeBulkButton } from "./ServerNotices";
import { DeviceRemoveButton } from "./devices";
import { makeStyles } from "@material-ui/core/styles";
const redirect = (basePath, id, data) => {
return {
pathname: "/importcsv",
};
};
const useStyles = makeStyles({
small: {
height: "40px",
@@ -71,27 +80,44 @@ const UserListActions = ({
maxResults,
total,
...rest
}) => (
<TopToolbar className={className} {...sanitizeListRestProps(rest)}>
{filters &&
cloneElement(filters, {
resource,
showFilter,
displayedFilters,
filterValues,
context: "button",
})}
<CreateButton basePath={basePath} />
<ExportButton
disabled={total === 0}
resource={resource}
sort={currentSort}
filter={{ ...filterValues, ...permanentFilter }}
exporter={exporter}
maxResults={maxResults}
/>
</TopToolbar>
);
}) => {
const redirectTo = useRedirect();
return (
<TopToolbar className={className} {...sanitizeListRestProps(rest)}>
{filters &&
cloneElement(filters, {
resource,
showFilter,
displayedFilters,
filterValues,
context: "button",
})}
<CreateButton basePath={basePath} />
<ExportButton
disabled={total === 0}
resource={resource}
sort={currentSort}
filter={{ ...filterValues, ...permanentFilter }}
exporter={exporter}
maxResults={maxResults}
/>
{/* Add your custom actions */}
<Button
onClick={() => {
redirectTo(redirect);
}}
label="CSV Import"
>
<GetAppIcon style={{ transform: "rotate(180deg)", fontSize: "20" }} />
</Button>
</TopToolbar>
);
};
UserListActions.defaultProps = {
selectedIds: [],
onUnselectItems: () => null,
};
const UserPagination = props => (
<Pagination {...props} rowsPerPageOptions={[10, 25, 50, 100, 500, 1000]} />
@@ -160,6 +186,47 @@ const validateUser = regex(
"synapseadmin.users.invalid_user_id"
);
export function generateRandomUser() {
const homeserver = localStorage.getItem("home_server");
const user_id =
"@" +
Array(8)
.fill("0123456789abcdefghijklmnopqrstuvwxyz")
.map(
x =>
x[
Math.floor(
(crypto.getRandomValues(new Uint32Array(1))[0] /
(0xffffffff + 1)) *
x.length
)
]
)
.join("") +
":" +
homeserver;
const password = Array(20)
.fill(
"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz~!@-#$"
)
.map(
x =>
x[
Math.floor(
(crypto.getRandomValues(new Uint32Array(1))[0] / (0xffffffff + 1)) *
x.length
)
]
)
.join("");
return {
id: user_id,
password: password,
};
}
const UserEditToolbar = props => {
const translate = useTranslate();
return (