parent
3d4f45d070
commit
e5d2d895d9
|
@ -0,0 +1,674 @@
|
||||
import React, { useState } from "react";
|
||||
import {
|
||||
Button as ReactAdminButton,
|
||||
useDataProvider,
|
||||
useNotify,
|
||||
Title,
|
||||
} from "react-admin";
|
||||
import { parse as parseCsv, unparse as unparseCsv } from "papaparse";
|
||||
import GetAppIcon from "@material-ui/icons/GetApp";
|
||||
import {
|
||||
Button,
|
||||
Card,
|
||||
CardActions,
|
||||
CardContent,
|
||||
CardHeader,
|
||||
FormControlLabel,
|
||||
Checkbox,
|
||||
NativeSelect,
|
||||
} from "@material-ui/core";
|
||||
import { useTranslate } from "ra-core";
|
||||
import Container from "@material-ui/core/Container/Container";
|
||||
import { generateRandomUser } from "./users";
|
||||
|
||||
const LOGGING = true;
|
||||
|
||||
export const ImportButton = ({ label, variant = "text" }) => {
|
||||
return (
|
||||
<ReactAdminButton
|
||||
color="primary"
|
||||
component="span"
|
||||
variant={variant}
|
||||
label={label}
|
||||
>
|
||||
<GetAppIcon style={{ transform: "rotate(180deg)", fontSize: "20" }} />
|
||||
</ReactAdminButton>
|
||||
);
|
||||
};
|
||||
|
||||
const expectedFields = ["id", "displayname"].sort();
|
||||
const optionalFields = [
|
||||
"user_type",
|
||||
"guest",
|
||||
"admin",
|
||||
"deactivated",
|
||||
"avatar_url",
|
||||
"password",
|
||||
].sort();
|
||||
|
||||
function TranslatableOption({ value, text }) {
|
||||
const translate = useTranslate();
|
||||
return <option value={value}>{translate(text)}</option>;
|
||||
}
|
||||
|
||||
const FilePicker = props => {
|
||||
const [values, setValues] = useState(null);
|
||||
const [error, setError] = useState(null);
|
||||
const [stats, setStats] = useState(null);
|
||||
const [dryRun, setDryRun] = useState(true);
|
||||
|
||||
const [progress, setProgress] = useState(null);
|
||||
|
||||
const [importResults, setImportResults] = useState(null);
|
||||
const [skippedRecords, setSkippedRecords] = useState(null);
|
||||
|
||||
const [conflictMode, setConflictMode] = useState("stop");
|
||||
const [passwordMode, setPasswordMode] = useState(true);
|
||||
const [useridMode, setUseridMode] = useState("ignore");
|
||||
|
||||
const translate = useTranslate();
|
||||
const notify = useNotify();
|
||||
|
||||
const dataProvider = useDataProvider();
|
||||
|
||||
const onFileChange = async e => {
|
||||
if (progress !== null) return;
|
||||
|
||||
setValues(null);
|
||||
setError(null);
|
||||
setStats(null);
|
||||
setImportResults(null);
|
||||
const file = e.target.files ? e.target.files[0] : null;
|
||||
/* Let's refuse some unreasonably big files instead of freezing
|
||||
* up the browser */
|
||||
if (file.size > 100000000) {
|
||||
const message = translate("import_users.errors.unreasonably_big", {
|
||||
size: (file.size / (1024 * 1024)).toFixed(2),
|
||||
});
|
||||
notify(message);
|
||||
setError(message);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
parseCsv(file, {
|
||||
header: true,
|
||||
skipEmptyLines: true /* especially for a final EOL in the csv file */,
|
||||
complete: result => {
|
||||
if (result.error) {
|
||||
setError(result.error);
|
||||
}
|
||||
/* Papaparse is very lenient, we may be able to salvage
|
||||
* the data in the file. */
|
||||
verifyCsv(result, { setValues, setStats, setError });
|
||||
},
|
||||
});
|
||||
} catch {
|
||||
setError(true);
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
const verifyCsv = (
|
||||
{ data, meta, errors },
|
||||
{ setValues, setStats, setError }
|
||||
) => {
|
||||
/* First, verify the presence of required fields */
|
||||
let eF = Array.from(expectedFields);
|
||||
let oF = Array.from(optionalFields);
|
||||
|
||||
meta.fields.forEach(name => {
|
||||
if (eF.includes(name)) {
|
||||
eF = eF.filter(v => v !== name);
|
||||
}
|
||||
if (oF.includes(name)) {
|
||||
oF = oF.filter(v => v !== name);
|
||||
}
|
||||
});
|
||||
|
||||
if (eF.length !== 0) {
|
||||
setError(
|
||||
translate("import_users.error.required_field", { field: eF[0] })
|
||||
);
|
||||
return false;
|
||||
}
|
||||
|
||||
// XXX after deciding on how "name" and friends should be handled below,
|
||||
// this place will want changes, too.
|
||||
|
||||
/* Collect some stats to prevent sneaky csv files from adding admin
|
||||
users or something.
|
||||
*/
|
||||
let stats = {
|
||||
user_types: { default: 0 },
|
||||
is_guest: 0,
|
||||
admin: 0,
|
||||
deactivated: 0,
|
||||
password: 0,
|
||||
avatar_url: 0,
|
||||
id: 0,
|
||||
|
||||
total: data.length,
|
||||
};
|
||||
|
||||
data.forEach((line, idx) => {
|
||||
if (line.user_type === undefined || line.user_type === "") {
|
||||
stats.user_types.default++;
|
||||
} else {
|
||||
stats.user_types[line.user_type] += 1;
|
||||
}
|
||||
/* XXX correct the csv export that react-admin offers for the users
|
||||
* resource so it gives sensible field names and doesn't duplicate
|
||||
* id as "name"?
|
||||
*/
|
||||
if (meta.fields.includes("name")) {
|
||||
delete line.name;
|
||||
}
|
||||
if (meta.fields.includes("user_type")) {
|
||||
delete line.user_type;
|
||||
}
|
||||
if (meta.fields.includes("is_admin")) {
|
||||
line.admin = line.is_admin;
|
||||
delete line.is_admin;
|
||||
}
|
||||
|
||||
["is_guest", "admin", "deactivated"].forEach(f => {
|
||||
if (line[f] === "true") {
|
||||
stats[f]++;
|
||||
line[f] = true; // we need true booleans instead of strings
|
||||
} else {
|
||||
if (line[f] !== "false" && line[f] !== "") {
|
||||
errors.push(
|
||||
translate("import_users.error.invalid_value", {
|
||||
field: f,
|
||||
row: idx,
|
||||
})
|
||||
);
|
||||
}
|
||||
line[f] = false; // default values to false
|
||||
}
|
||||
});
|
||||
|
||||
if (line.password !== undefined && line.password !== "") {
|
||||
stats.password++;
|
||||
}
|
||||
|
||||
if (line.avatar_url !== undefined && line.avatar_url !== "") {
|
||||
stats.avatar_url++;
|
||||
}
|
||||
|
||||
if (line.id !== undefined && line.id !== "") {
|
||||
stats.id++;
|
||||
}
|
||||
});
|
||||
|
||||
if (errors.length > 0) {
|
||||
setError(errors);
|
||||
}
|
||||
setStats(stats);
|
||||
setValues(data);
|
||||
|
||||
return true;
|
||||
};
|
||||
|
||||
const runImport = async e => {
|
||||
if (progress !== null) {
|
||||
notify("import_users.errors.already_in_progress");
|
||||
return;
|
||||
}
|
||||
|
||||
const results = await doImport(
|
||||
dataProvider,
|
||||
values,
|
||||
conflictMode,
|
||||
passwordMode,
|
||||
useridMode,
|
||||
dryRun,
|
||||
setProgress,
|
||||
setError
|
||||
);
|
||||
setImportResults(results);
|
||||
// offer CSV download of skipped or errored records
|
||||
// (so that the user doesn't have to filter out successful
|
||||
// records manually when fixing stuff in the CSV)
|
||||
setSkippedRecords(unparseCsv(results.skippedRecords));
|
||||
if (LOGGING) console.log("Skipped records:");
|
||||
if (LOGGING) console.log(skippedRecords);
|
||||
};
|
||||
|
||||
// XXX every single one of the requests will restart the activity indicator
|
||||
// which doesn't look very good.
|
||||
|
||||
const doImport = async (
|
||||
dataProvider,
|
||||
data,
|
||||
conflictMode,
|
||||
passwordMode,
|
||||
useridMode,
|
||||
dryRun,
|
||||
setProgress,
|
||||
setError
|
||||
) => {
|
||||
let skippedRecords = [];
|
||||
let erroredRecords = [];
|
||||
let succeededRecords = [];
|
||||
let changeStats = {
|
||||
toAdmin: 0,
|
||||
toGuest: 0,
|
||||
toRegular: 0,
|
||||
replacedPassword: 0,
|
||||
};
|
||||
let entriesDone = 0;
|
||||
let entriesCount = data.length;
|
||||
try {
|
||||
setProgress({ done: entriesDone, limit: entriesCount });
|
||||
for (const entry of data) {
|
||||
let userRecord = {};
|
||||
let overwriteData = {};
|
||||
// No need to do a bunch of cryptographic random number getting if
|
||||
// we are using neither a generated password nor a generated user id.
|
||||
if (
|
||||
useridMode === "ignore" ||
|
||||
entry.id === undefined ||
|
||||
entry.password === undefined ||
|
||||
passwordMode === false
|
||||
) {
|
||||
overwriteData = generateRandomUser();
|
||||
// Ignoring IDs or the entry lacking an ID means we keep the
|
||||
// ID field in the overwrite data.
|
||||
if (!(useridMode === "ignore" || entry.id === undefined)) {
|
||||
delete overwriteData.id;
|
||||
}
|
||||
|
||||
// Not using passwords from the csv or this entry lacking a password
|
||||
// means we keep the password field in the overwrite data.
|
||||
if (
|
||||
!(
|
||||
passwordMode === false ||
|
||||
entry.password === undefined ||
|
||||
entry.password === ""
|
||||
)
|
||||
) {
|
||||
delete overwriteData.password;
|
||||
}
|
||||
}
|
||||
/* TODO record update stats (especially admin no -> yes, deactivated x -> !x, ... */
|
||||
Object.assign(userRecord, entry);
|
||||
Object.assign(userRecord, overwriteData);
|
||||
|
||||
/* For these modes we will consider the ID that's in the record.
|
||||
* If the mode is "stop", we will not continue adding more records, and
|
||||
* we will offer information on what was already added and what was
|
||||
* skipped.
|
||||
*
|
||||
* If the mode is "skip", we record the record for later, but don't
|
||||
* send it to the server.
|
||||
*
|
||||
* If the mode is "update", we change fields that are reasonable to
|
||||
* update.
|
||||
* - If the "password mode" is "true" (i.e. "use passwords from csv"):
|
||||
* - if the record has a password
|
||||
* - send the password along with the record
|
||||
* - if the record has no password
|
||||
* - generate a new password
|
||||
* - If the "password mode" is "false"
|
||||
* - never generate a new password to update existing users with
|
||||
*/
|
||||
|
||||
/* We just act as if there are no IDs in the CSV, so every user will be
|
||||
* created anew.
|
||||
* We do a simple retry loop so that an accidental hit on an existing ID
|
||||
* doesn't trip us up.
|
||||
*/
|
||||
if (LOGGING)
|
||||
console.log(
|
||||
"will check for existence of record " + JSON.stringify(userRecord)
|
||||
);
|
||||
let retries = 0;
|
||||
const submitRecord = recordData => {
|
||||
return dataProvider.getOne("users", { id: recordData.id }).then(
|
||||
async alreadyExists => {
|
||||
if (LOGGING) console.log("already existed");
|
||||
|
||||
if (useridMode === "update" || conflictMode === "skip") {
|
||||
skippedRecords.push(recordData);
|
||||
} else if (conflictMode === "stop") {
|
||||
throw new Error(
|
||||
translate("import_users.error.id_exits", {
|
||||
id: recordData.id,
|
||||
})
|
||||
);
|
||||
} else {
|
||||
const overwriteData = generateRandomUser();
|
||||
const newRecordData = Object.assign({}, recordData, {
|
||||
id: overwriteData.id,
|
||||
});
|
||||
retries++;
|
||||
if (retries > 512) {
|
||||
console.warn("retry loop got stuck? pathological situation?");
|
||||
skippedRecords.push(recordData);
|
||||
} else {
|
||||
await submitRecord(newRecordData);
|
||||
}
|
||||
}
|
||||
},
|
||||
async okToSubmit => {
|
||||
if (LOGGING)
|
||||
console.log(
|
||||
"OK to create record " +
|
||||
recordData.id +
|
||||
" (" +
|
||||
recordData.displayname +
|
||||
")."
|
||||
);
|
||||
|
||||
if (!dryRun) {
|
||||
await dataProvider.create("users", { data: recordData });
|
||||
}
|
||||
succeededRecords.push(recordData);
|
||||
}
|
||||
);
|
||||
};
|
||||
|
||||
await submitRecord(userRecord);
|
||||
entriesDone++;
|
||||
setProgress({ done: entriesDone, limit: data.length });
|
||||
}
|
||||
|
||||
setProgress(null);
|
||||
} catch (e) {
|
||||
setError(
|
||||
translate("import_users.error.at_entry", {
|
||||
entry: entriesDone + 1,
|
||||
message: e.message,
|
||||
})
|
||||
);
|
||||
setProgress(null);
|
||||
}
|
||||
return {
|
||||
skippedRecords,
|
||||
erroredRecords,
|
||||
succeededRecords,
|
||||
totalRecordCount: entriesCount,
|
||||
changeStats,
|
||||
wasDryRun: dryRun,
|
||||
};
|
||||
};
|
||||
|
||||
const downloadSkippedRecords = () => {
|
||||
const element = document.createElement("a");
|
||||
console.log(skippedRecords);
|
||||
const file = new Blob([skippedRecords], {
|
||||
type: "text/comma-separated-values",
|
||||
});
|
||||
element.href = URL.createObjectURL(file);
|
||||
element.download = "skippedRecords.csv";
|
||||
document.body.appendChild(element); // Required for this to work in FireFox
|
||||
element.click();
|
||||
};
|
||||
|
||||
const onConflictModeChanged = async e => {
|
||||
if (progress !== null) {
|
||||
return;
|
||||
}
|
||||
|
||||
const value = e.target.value;
|
||||
setConflictMode(value);
|
||||
};
|
||||
|
||||
const onPasswordModeChange = e => {
|
||||
if (progress !== null) {
|
||||
return;
|
||||
}
|
||||
|
||||
setPasswordMode(e.target.checked);
|
||||
};
|
||||
|
||||
const onUseridModeChanged = async e => {
|
||||
if (progress !== null) {
|
||||
return;
|
||||
}
|
||||
|
||||
const value = e.target.value;
|
||||
setUseridMode(value);
|
||||
};
|
||||
|
||||
const onDryRunModeChanged = ev => {
|
||||
if (progress !== null) {
|
||||
return;
|
||||
}
|
||||
setDryRun(ev.target.checked);
|
||||
};
|
||||
|
||||
// render individual small components
|
||||
|
||||
const statsCards = stats &&
|
||||
!importResults && [
|
||||
<Container>
|
||||
<CardHeader
|
||||
title={translate("import_users.cards.importstats.header")}
|
||||
/>
|
||||
<CardContent>
|
||||
<div>
|
||||
{translate(
|
||||
"import_users.cards.importstats.users_total",
|
||||
stats.total
|
||||
)}
|
||||
</div>
|
||||
<div>
|
||||
{translate(
|
||||
"import_users.cards.importstats.guest_count",
|
||||
stats.is_guest
|
||||
)}
|
||||
</div>
|
||||
<div>
|
||||
{translate(
|
||||
"import_users.cards.importstats.admin_count",
|
||||
stats.admin
|
||||
)}
|
||||
</div>
|
||||
</CardContent>
|
||||
</Container>,
|
||||
<Container>
|
||||
<CardHeader title={translate("import_users.cards.ids.header")} />
|
||||
<CardContent>
|
||||
<div>
|
||||
{stats.id === stats.total
|
||||
? translate("import_users.cards.ids.all_ids_present")
|
||||
: translate("import_users.cards.ids.count_ids_present", stats.id)}
|
||||
</div>
|
||||
{stats.id > 0 ? (
|
||||
<div>
|
||||
<NativeSelect
|
||||
onChange={onUseridModeChanged}
|
||||
value={useridMode}
|
||||
enabled={(progress !== null).toString()}
|
||||
>
|
||||
<TranslatableOption
|
||||
value="ignore"
|
||||
text="import_users.cards.ids.mode.ignore"
|
||||
/>
|
||||
<TranslatableOption
|
||||
value="update"
|
||||
text="import_users.cards.ids.mode.update"
|
||||
/>
|
||||
</NativeSelect>
|
||||
</div>
|
||||
) : (
|
||||
""
|
||||
)}
|
||||
</CardContent>
|
||||
</Container>,
|
||||
<Container>
|
||||
<CardHeader title={translate("import_users.cards.passwords.header")} />
|
||||
<CardContent>
|
||||
<div>
|
||||
{stats.password === stats.total
|
||||
? translate("import_users.cards.passwords.all_passwords_present")
|
||||
: translate(
|
||||
"import_users.cards.passwords.count_passwords_present",
|
||||
stats.password
|
||||
)}
|
||||
</div>
|
||||
{stats.password > 0 ? (
|
||||
<div>
|
||||
<FormControlLabel
|
||||
control={
|
||||
<Checkbox
|
||||
checked={passwordMode}
|
||||
enabled={(progress !== null).toString()}
|
||||
onChange={onPasswordModeChange}
|
||||
/>
|
||||
}
|
||||
label={translate("import_users.cards.passwords.use_passwords")}
|
||||
/>
|
||||
</div>
|
||||
) : (
|
||||
""
|
||||
)}
|
||||
</CardContent>
|
||||
</Container>,
|
||||
];
|
||||
|
||||
let conflictCards = stats && !importResults && (
|
||||
<Container>
|
||||
<CardHeader title={translate("import_users.cards.conflicts.header")} />
|
||||
<CardContent>
|
||||
<div>
|
||||
<NativeSelect
|
||||
onChange={onConflictModeChanged}
|
||||
value={conflictMode}
|
||||
enabled={(progress !== null).toString()}
|
||||
>
|
||||
<TranslatableOption
|
||||
value="stop"
|
||||
text="import_users.cards.conflicts.mode.stop"
|
||||
/>
|
||||
<TranslatableOption
|
||||
value="skip"
|
||||
text="import_users.cards.conflicts.mode.skip"
|
||||
/>
|
||||
</NativeSelect>
|
||||
</div>
|
||||
</CardContent>
|
||||
</Container>
|
||||
);
|
||||
|
||||
let errorCards = error && (
|
||||
<Container>
|
||||
<CardHeader title={translate("import_users.error.error")} />
|
||||
<CardContent>
|
||||
{(Array.isArray(error) ? error : [error]).map(e => (
|
||||
<div>{e}</div>
|
||||
))}
|
||||
</CardContent>
|
||||
</Container>
|
||||
);
|
||||
|
||||
let uploadCard = !importResults && (
|
||||
<Container>
|
||||
<CardHeader title={translate("import_users.cards.upload.header")} />
|
||||
<CardContent>
|
||||
{translate("import_users.cards.upload.explanation")}
|
||||
<a href="/data/example.csv">example.csv</a>
|
||||
<br />
|
||||
<br />
|
||||
<input
|
||||
type="file"
|
||||
onChange={onFileChange}
|
||||
enabled={(progress !== null).toString()}
|
||||
/>
|
||||
</CardContent>
|
||||
</Container>
|
||||
);
|
||||
|
||||
let resultsCard = importResults && (
|
||||
<CardContent>
|
||||
<CardHeader title={translate("import_users.cards.results.header")} />
|
||||
<div>
|
||||
{translate(
|
||||
"import_users.cards.results.total",
|
||||
importResults.totalRecordCount
|
||||
)}
|
||||
<br />
|
||||
{translate(
|
||||
"import_users.cards.results.successful",
|
||||
importResults.succeededRecords.length
|
||||
)}
|
||||
<br />
|
||||
{importResults.skippedRecords.length
|
||||
? [
|
||||
translate(
|
||||
"import_users.cards.results.skipped",
|
||||
importResults.skippedRecords.length
|
||||
),
|
||||
<div>
|
||||
<button onClick={downloadSkippedRecords}>
|
||||
{translate("import_users.cards.results.download_skipped")}
|
||||
</button>
|
||||
</div>,
|
||||
<br />,
|
||||
]
|
||||
: ""}
|
||||
{importResults.erroredRecords.length
|
||||
? [
|
||||
translate(
|
||||
"import_users.cards.results.skipped",
|
||||
importResults.erroredRecords.length
|
||||
),
|
||||
<br />,
|
||||
]
|
||||
: ""}
|
||||
<br />
|
||||
{importResults.wasDryRun && [
|
||||
translate("import_users.cards.results.simulated_only"),
|
||||
<br />,
|
||||
]}
|
||||
</div>
|
||||
</CardContent>
|
||||
);
|
||||
|
||||
let startImportCard =
|
||||
!values || values.length === 0 || importResults ? undefined : (
|
||||
<CardActions>
|
||||
<FormControlLabel
|
||||
control={
|
||||
<Checkbox
|
||||
checked={dryRun}
|
||||
onChange={onDryRunModeChanged}
|
||||
enabled={(progress !== null).toString()}
|
||||
/>
|
||||
}
|
||||
label={translate("import_users.cards.startImport.simulate_only")}
|
||||
/>
|
||||
<Button
|
||||
size="large"
|
||||
onClick={runImport}
|
||||
enabled={(progress !== null).toString()}
|
||||
>
|
||||
{translate("import_users.cards.startImport.run_import")}
|
||||
</Button>
|
||||
{progress !== null ? (
|
||||
<div>
|
||||
{progress.done} of {progress.limit} done
|
||||
</div>
|
||||
) : null}
|
||||
</CardActions>
|
||||
);
|
||||
|
||||
let allCards = [];
|
||||
if (uploadCard) allCards.push(uploadCard);
|
||||
if (errorCards) allCards.push(errorCards);
|
||||
if (conflictCards) allCards.push(conflictCards);
|
||||
if (statsCards) allCards.push(...statsCards);
|
||||
if (startImportCard) allCards.push(startImportCard);
|
||||
if (resultsCard) allCards.push(resultsCard);
|
||||
|
||||
let cardContainer = <Card>{allCards}</Card>;
|
||||
|
||||
return [
|
||||
<Title defaultTitle={translate("import_users.title")} />,
|
||||
cardContainer,
|
||||
];
|
||||
};
|
||||
|
||||
export const ImportFeature = FilePicker;
|
@ -0,0 +1,39 @@
|
||||
// in src/Menu.js
|
||||
import * as React from "react";
|
||||
import { useSelector } from "react-redux";
|
||||
import { useMediaQuery } from "@material-ui/core";
|
||||
import { MenuItemLink, getResources } from "react-admin";
|
||||
import DefaultIcon from "@material-ui/icons/ViewList";
|
||||
import LabelIcon from "@material-ui/icons/Label";
|
||||
|
||||
const Menu = ({ onMenuClick, logout }) => {
|
||||
const isXSmall = useMediaQuery(theme => theme.breakpoints.down("xs"));
|
||||
const open = useSelector(state => state.admin.ui.sidebarOpen);
|
||||
const resources = useSelector(getResources);
|
||||
return (
|
||||
<div>
|
||||
{resources.map(resource => (
|
||||
<MenuItemLink
|
||||
key={resource.name}
|
||||
to={`/${resource.name}`}
|
||||
primaryText={
|
||||
(resource.options && resource.options.label) || resource.name
|
||||
}
|
||||
leftIcon={resource.icon ? <resource.icon /> : <DefaultIcon />}
|
||||
onClick={onMenuClick}
|
||||
sidebarIsOpen={open}
|
||||
/>
|
||||
))}
|
||||
<MenuItemLink
|
||||
to="/custom-route"
|
||||
primaryText="Miscellaneous"
|
||||
leftIcon={<LabelIcon />}
|
||||
onClick={onMenuClick}
|
||||
sidebarIsOpen={open}
|
||||
/>
|
||||
{isXSmall && logout}
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
export default Menu;
|
Loading…
Reference in new issue