diff --git a/package.json b/package.json index 09db13f..48d584a 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "dependencies": { "prop-types": "^15.7.2", "ra-language-german": "^2.1.2", + "papaparse": "^5.2.0", "react": "^16.13.1", "react-admin": "^3.7.0", "react-dom": "^16.13.1", diff --git a/public/data/example.csv b/public/data/example.csv new file mode 100644 index 0000000..02933b0 --- /dev/null +++ b/public/data/example.csv @@ -0,0 +1,3 @@ +id,displayname,password,is_guest,admin,deactivated +@testuser22:example.org,Jane Doe,secretpassword,false,true,false +,John Doe,,false,false,false diff --git a/src/App.js b/src/App.js index 8696936..6ad0b22 100644 --- a/src/App.js +++ b/src/App.js @@ -8,6 +8,8 @@ import { RoomList, RoomShow } from "./components/rooms"; import LoginPage from "./components/LoginPage"; import UserIcon from "@material-ui/icons/Group"; import { ViewListIcon as RoomIcon } from "@material-ui/icons/ViewList"; +import { ImportFeature } from "./components/ImportFeature"; +import { Route } from "react-router-dom"; import germanMessages from "./i18n/de"; import englishMessages from "./i18n/en"; @@ -27,6 +29,9 @@ const App = () => ( authProvider={authProvider} dataProvider={dataProvider} i18nProvider={i18nProvider} + customRoutes={[ + , + ]} > { + return ( + + + + ); +}; + +const expectedFields = ["id", "displayname"].sort(); +const optionalFields = [ + "user_type", + "guest", + "admin", + "deactivated", + "avatar_url", + "password", +].sort(); + +function TranslatableOption({ value, text }) { + const translate = useTranslate(); + return ; +} + +const FilePicker = props => { + const [values, setValues] = useState(null); + const [error, setError] = useState(null); + const [stats, setStats] = useState(null); + const [dryRun, setDryRun] = useState(true); + + const [progress, setProgress] = useState(null); + + const [importResults, setImportResults] = useState(null); + const [skippedRecords, setSkippedRecords] = useState(null); + + const [conflictMode, setConflictMode] = useState("stop"); + const [passwordMode, setPasswordMode] = useState(true); + const [useridMode, setUseridMode] = useState("ignore"); + + const translate = useTranslate(); + const notify = useNotify(); + + const dataProvider = useDataProvider(); + + const onFileChange = async e => { + if (progress !== null) return; + + setValues(null); + setError(null); + setStats(null); + setImportResults(null); + const file = e.target.files ? e.target.files[0] : null; + /* Let's refuse some unreasonably big files instead of freezing + * up the browser */ + if (file.size > 100000000) { + const message = translate("import_users.errors.unreasonably_big", { + size: (file.size / (1024 * 1024)).toFixed(2), + }); + notify(message); + setError(message); + return; + } + try { + parseCsv(file, { + header: true, + skipEmptyLines: true /* especially for a final EOL in the csv file */, + complete: result => { + if (result.error) { + setError(result.error); + } + /* Papaparse is very lenient, we may be able to salvage + * the data in the file. */ + verifyCsv(result, { setValues, setStats, setError }); + }, + }); + } catch { + setError(true); + return null; + } + }; + + const verifyCsv = ( + { data, meta, errors }, + { setValues, setStats, setError } + ) => { + /* First, verify the presence of required fields */ + let eF = Array.from(expectedFields); + let oF = Array.from(optionalFields); + + meta.fields.forEach(name => { + if (eF.includes(name)) { + eF = eF.filter(v => v !== name); + } + if (oF.includes(name)) { + oF = oF.filter(v => v !== name); + } + }); + + if (eF.length !== 0) { + setError( + translate("import_users.error.required_field", { field: eF[0] }) + ); + return false; + } + + // XXX after deciding on how "name" and friends should be handled below, + // this place will want changes, too. + + /* Collect some stats to prevent sneaky csv files from adding admin + users or something. + */ + let stats = { + user_types: { default: 0 }, + is_guest: 0, + admin: 0, + deactivated: 0, + password: 0, + avatar_url: 0, + id: 0, + + total: data.length, + }; + + data.forEach((line, idx) => { + if (line.user_type === undefined || line.user_type === "") { + stats.user_types.default++; + } else { + stats.user_types[line.user_type] += 1; + } + /* XXX correct the csv export that react-admin offers for the users + * resource so it gives sensible field names and doesn't duplicate + * id as "name"? + */ + if (meta.fields.includes("name")) { + delete line.name; + } + if (meta.fields.includes("user_type")) { + delete line.user_type; + } + if (meta.fields.includes("is_admin")) { + line.admin = line.is_admin; + delete line.is_admin; + } + + ["is_guest", "admin", "deactivated"].forEach(f => { + if (line[f] === "true") { + stats[f]++; + line[f] = true; // we need true booleans instead of strings + } else { + if (line[f] !== "false" && line[f] !== "") { + errors.push( + translate("import_users.error.invalid_value", { + field: f, + row: idx, + }) + ); + } + line[f] = false; // default values to false + } + }); + + if (line.password !== undefined && line.password !== "") { + stats.password++; + } + + if (line.avatar_url !== undefined && line.avatar_url !== "") { + stats.avatar_url++; + } + + if (line.id !== undefined && line.id !== "") { + stats.id++; + } + }); + + if (errors.length > 0) { + setError(errors); + } + setStats(stats); + setValues(data); + + return true; + }; + + const runImport = async e => { + if (progress !== null) { + notify("import_users.errors.already_in_progress"); + return; + } + + const results = await doImport( + dataProvider, + values, + conflictMode, + passwordMode, + useridMode, + dryRun, + setProgress, + setError + ); + setImportResults(results); + // offer CSV download of skipped or errored records + // (so that the user doesn't have to filter out successful + // records manually when fixing stuff in the CSV) + setSkippedRecords(unparseCsv(results.skippedRecords)); + if (LOGGING) console.log("Skipped records:"); + if (LOGGING) console.log(skippedRecords); + }; + + // XXX every single one of the requests will restart the activity indicator + // which doesn't look very good. + + const doImport = async ( + dataProvider, + data, + conflictMode, + passwordMode, + useridMode, + dryRun, + setProgress, + setError + ) => { + let skippedRecords = []; + let erroredRecords = []; + let succeededRecords = []; + let changeStats = { + toAdmin: 0, + toGuest: 0, + toRegular: 0, + replacedPassword: 0, + }; + let entriesDone = 0; + let entriesCount = data.length; + try { + setProgress({ done: entriesDone, limit: entriesCount }); + for (const entry of data) { + let userRecord = {}; + let overwriteData = {}; + // No need to do a bunch of cryptographic random number getting if + // we are using neither a generated password nor a generated user id. + if ( + useridMode === "ignore" || + entry.id === undefined || + entry.password === undefined || + passwordMode === false + ) { + overwriteData = generateRandomUser(); + // Ignoring IDs or the entry lacking an ID means we keep the + // ID field in the overwrite data. + if (!(useridMode === "ignore" || entry.id === undefined)) { + delete overwriteData.id; + } + + // Not using passwords from the csv or this entry lacking a password + // means we keep the password field in the overwrite data. + if ( + !( + passwordMode === false || + entry.password === undefined || + entry.password === "" + ) + ) { + delete overwriteData.password; + } + } + /* TODO record update stats (especially admin no -> yes, deactivated x -> !x, ... */ + Object.assign(userRecord, entry); + Object.assign(userRecord, overwriteData); + + /* For these modes we will consider the ID that's in the record. + * If the mode is "stop", we will not continue adding more records, and + * we will offer information on what was already added and what was + * skipped. + * + * If the mode is "skip", we record the record for later, but don't + * send it to the server. + * + * If the mode is "update", we change fields that are reasonable to + * update. + * - If the "password mode" is "true" (i.e. "use passwords from csv"): + * - if the record has a password + * - send the password along with the record + * - if the record has no password + * - generate a new password + * - If the "password mode" is "false" + * - never generate a new password to update existing users with + */ + + /* We just act as if there are no IDs in the CSV, so every user will be + * created anew. + * We do a simple retry loop so that an accidental hit on an existing ID + * doesn't trip us up. + */ + if (LOGGING) + console.log( + "will check for existence of record " + JSON.stringify(userRecord) + ); + let retries = 0; + const submitRecord = recordData => { + return dataProvider.getOne("users", { id: recordData.id }).then( + async alreadyExists => { + if (LOGGING) console.log("already existed"); + + if (useridMode === "update" || conflictMode === "skip") { + skippedRecords.push(recordData); + } else if (conflictMode === "stop") { + throw new Error( + translate("import_users.error.id_exits", { + id: recordData.id, + }) + ); + } else { + const overwriteData = generateRandomUser(); + const newRecordData = Object.assign({}, recordData, { + id: overwriteData.id, + }); + retries++; + if (retries > 512) { + console.warn("retry loop got stuck? pathological situation?"); + skippedRecords.push(recordData); + } else { + await submitRecord(newRecordData); + } + } + }, + async okToSubmit => { + if (LOGGING) + console.log( + "OK to create record " + + recordData.id + + " (" + + recordData.displayname + + ")." + ); + + if (!dryRun) { + await dataProvider.create("users", { data: recordData }); + } + succeededRecords.push(recordData); + } + ); + }; + + await submitRecord(userRecord); + entriesDone++; + setProgress({ done: entriesDone, limit: data.length }); + } + + setProgress(null); + } catch (e) { + setError( + translate("import_users.error.at_entry", { + entry: entriesDone + 1, + message: e.message, + }) + ); + setProgress(null); + } + return { + skippedRecords, + erroredRecords, + succeededRecords, + totalRecordCount: entriesCount, + changeStats, + wasDryRun: dryRun, + }; + }; + + const downloadSkippedRecords = () => { + const element = document.createElement("a"); + console.log(skippedRecords); + const file = new Blob([skippedRecords], { + type: "text/comma-separated-values", + }); + element.href = URL.createObjectURL(file); + element.download = "skippedRecords.csv"; + document.body.appendChild(element); // Required for this to work in FireFox + element.click(); + }; + + const onConflictModeChanged = async e => { + if (progress !== null) { + return; + } + + const value = e.target.value; + setConflictMode(value); + }; + + const onPasswordModeChange = e => { + if (progress !== null) { + return; + } + + setPasswordMode(e.target.checked); + }; + + const onUseridModeChanged = async e => { + if (progress !== null) { + return; + } + + const value = e.target.value; + setUseridMode(value); + }; + + const onDryRunModeChanged = ev => { + if (progress !== null) { + return; + } + setDryRun(ev.target.checked); + }; + + // render individual small components + + const statsCards = stats && + !importResults && [ + + + +
+ {translate( + "import_users.cards.importstats.users_total", + stats.total + )} +
+
+ {translate( + "import_users.cards.importstats.guest_count", + stats.is_guest + )} +
+
+ {translate( + "import_users.cards.importstats.admin_count", + stats.admin + )} +
+ + , + + + +
+ {stats.id === stats.total + ? translate("import_users.cards.ids.all_ids_present") + : translate("import_users.cards.ids.count_ids_present", stats.id)} +
+ {stats.id > 0 ? ( +
+ + + + +
+ ) : ( + "" + )} + + , + + + +
+ {stats.password === stats.total + ? translate("import_users.cards.passwords.all_passwords_present") + : translate( + "import_users.cards.passwords.count_passwords_present", + stats.password + )} +
+ {stats.password > 0 ? ( +
+ + } + label={translate("import_users.cards.passwords.use_passwords")} + /> +
+ ) : ( + "" + )} + + , + ]; + + let conflictCards = stats && !importResults && ( + + + +
+ + + + +
+ + + ); + + let errorCards = error && ( + + + + {(Array.isArray(error) ? error : [error]).map(e => ( +
{e}
+ ))} + + + ); + + let uploadCard = !importResults && ( + + + + {translate("import_users.cards.upload.explanation")} + example.csv +
+
+ + + + ); + + let resultsCard = importResults && ( + + +
+ {translate( + "import_users.cards.results.total", + importResults.totalRecordCount + )} +
+ {translate( + "import_users.cards.results.successful", + importResults.succeededRecords.length + )} +
+ {importResults.skippedRecords.length + ? [ + translate( + "import_users.cards.results.skipped", + importResults.skippedRecords.length + ), +
+ +
, +
, + ] + : ""} + {importResults.erroredRecords.length + ? [ + translate( + "import_users.cards.results.skipped", + importResults.erroredRecords.length + ), +
, + ] + : ""} +
+ {importResults.wasDryRun && [ + translate("import_users.cards.results.simulated_only"), +
, + ]} +
+ + ); + + let startImportCard = + !values || values.length === 0 || importResults ? undefined : ( + + + } + label={translate("import_users.cards.startImport.simulate_only")} + /> + + {progress !== null ? ( +
+ {progress.done} of {progress.limit} done +
+ ) : null} + + ); + + let allCards = []; + if (uploadCard) allCards.push(uploadCard); + if (errorCards) allCards.push(errorCards); + if (conflictCards) allCards.push(conflictCards); + if (statsCards) allCards.push(...statsCards); + if (startImportCard) allCards.push(startImportCard); + if (resultsCard) allCards.push(resultsCard); + + let cardContainer = {allCards}; + + return [ + , + cardContainer, + ]; +}; + +export const ImportFeature = FilePicker; diff --git a/src/components/Menu.js b/src/components/Menu.js new file mode 100644 index 0000000..39e28c8 --- /dev/null +++ b/src/components/Menu.js @@ -0,0 +1,39 @@ +// in src/Menu.js +import * as React from "react"; +import { useSelector } from "react-redux"; +import { useMediaQuery } from "@material-ui/core"; +import { MenuItemLink, getResources } from "react-admin"; +import DefaultIcon from "@material-ui/icons/ViewList"; +import LabelIcon from "@material-ui/icons/Label"; + +const Menu = ({ onMenuClick, logout }) => { + const isXSmall = useMediaQuery(theme => theme.breakpoints.down("xs")); + const open = useSelector(state => state.admin.ui.sidebarOpen); + const resources = useSelector(getResources); + return ( + <div> + {resources.map(resource => ( + <MenuItemLink + key={resource.name} + to={`/${resource.name}`} + primaryText={ + (resource.options && resource.options.label) || resource.name + } + leftIcon={resource.icon ? <resource.icon /> : <DefaultIcon />} + onClick={onMenuClick} + sidebarIsOpen={open} + /> + ))} + <MenuItemLink + to="/custom-route" + primaryText="Miscellaneous" + leftIcon={<LabelIcon />} + onClick={onMenuClick} + sidebarIsOpen={open} + /> + {isXSmall && logout} + </div> + ); +}; + +export default Menu; diff --git a/src/components/users.js b/src/components/users.js index d27cde5..827dca0 100644 --- a/src/components/users.js +++ b/src/components/users.js @@ -3,10 +3,12 @@ import Avatar from "@material-ui/core/Avatar"; import PersonPinIcon from "@material-ui/icons/PersonPin"; import ContactMailIcon from "@material-ui/icons/ContactMail"; import DevicesIcon from "@material-ui/icons/Devices"; +import GetAppIcon from "@material-ui/icons/GetApp"; import SettingsInputComponentIcon from "@material-ui/icons/SettingsInputComponent"; import { ArrayInput, ArrayField, + Button, Datagrid, DateField, Create, @@ -31,6 +33,7 @@ import { DeleteButton, SaveButton, regex, + useRedirect, useTranslate, Pagination, CreateButton, @@ -42,6 +45,12 @@ import { ServerNoticeButton, ServerNoticeBulkButton } from "./ServerNotices"; import { DeviceRemoveButton } from "./devices"; import { makeStyles } from "@material-ui/core/styles"; +const redirect = (basePath, id, data) => { + return { + pathname: "/importcsv", + }; +}; + const useStyles = makeStyles({ small: { height: "40px", @@ -71,27 +80,44 @@ const UserListActions = ({ maxResults, total, ...rest -}) => ( - <TopToolbar className={className} {...sanitizeListRestProps(rest)}> - {filters && - cloneElement(filters, { - resource, - showFilter, - displayedFilters, - filterValues, - context: "button", - })} - <CreateButton basePath={basePath} /> - <ExportButton - disabled={total === 0} - resource={resource} - sort={currentSort} - filter={{ ...filterValues, ...permanentFilter }} - exporter={exporter} - maxResults={maxResults} - /> - </TopToolbar> -); +}) => { + const redirectTo = useRedirect(); + return ( + <TopToolbar className={className} {...sanitizeListRestProps(rest)}> + {filters && + cloneElement(filters, { + resource, + showFilter, + displayedFilters, + filterValues, + context: "button", + })} + <CreateButton basePath={basePath} /> + <ExportButton + disabled={total === 0} + resource={resource} + sort={currentSort} + filter={{ ...filterValues, ...permanentFilter }} + exporter={exporter} + maxResults={maxResults} + /> + {/* Add your custom actions */} + <Button + onClick={() => { + redirectTo(redirect); + }} + label="CSV Import" + > + <GetAppIcon style={{ transform: "rotate(180deg)", fontSize: "20" }} /> + </Button> + </TopToolbar> + ); +}; + +UserListActions.defaultProps = { + selectedIds: [], + onUnselectItems: () => null, +}; const UserPagination = props => ( <Pagination {...props} rowsPerPageOptions={[10, 25, 50, 100, 500, 1000]} /> @@ -160,6 +186,47 @@ const validateUser = regex( "synapseadmin.users.invalid_user_id" ); +export function generateRandomUser() { + const homeserver = localStorage.getItem("home_server"); + const user_id = + "@" + + Array(8) + .fill("0123456789abcdefghijklmnopqrstuvwxyz") + .map( + x => + x[ + Math.floor( + (crypto.getRandomValues(new Uint32Array(1))[0] / + (0xffffffff + 1)) * + x.length + ) + ] + ) + .join("") + + ":" + + homeserver; + + const password = Array(20) + .fill( + "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz~!@-#$" + ) + .map( + x => + x[ + Math.floor( + (crypto.getRandomValues(new Uint32Array(1))[0] / (0xffffffff + 1)) * + x.length + ) + ] + ) + .join(""); + + return { + id: user_id, + password: password, + }; +} + const UserEditToolbar = props => { const translate = useTranslate(); return ( diff --git a/src/i18n/de.js b/src/i18n/de.js index be8b249..0342ec4 100644 --- a/src/i18n/de.js +++ b/src/i18n/de.js @@ -29,6 +29,74 @@ export default { }, }, }, + import_users: { + error: { + at_entry: "Bei Eintrag %{entry}: %{message}", + error: "Fehler", + required_field: "Pflichtfeld '%{field}' fehlt", + invalid_value: + "Ungültiger Wert in Zeile %{row}. Feld '%{field}' darf nur die Werte 'true' oder 'false' enthalten", + unreasonably_big: "Datei ist zu groß für den Import (%{size} Megabytes)", + already_in_progress: "Es läuft bereits ein Import", + id_exits: "ID %{id} existiert bereits", + }, + title: "Benutzer aus CSV importieren", + goToPdf: "Gehe zum PDF", + cards: { + importstats: { + header: "Benutzer importieren", + users_total: + "%{smart_count} Benutzer in der CSV Datei |||| %{smart_count} Benutzer in der CSV Datei", + guest_count: "%{smart_count} Gast |||| %{smart_count} Gäste", + admin_count: + "%{smart_count} Server Administrator |||| %{smart_count} Server Administratoren", + }, + conflicts: { + header: "Konfliktstrategie", + mode: { + stop: "Stoppe bei Fehlern", + skip: "Zeige Fehler und überspringe fehlerhafte Einträge", + }, + }, + ids: { + header: "IDs", + all_ids_present: "IDs in jedem Eintrag vorhanden", + count_ids_present: + "%{smart_count} Eintrag mit ID |||| %{smart_count} Einträge mit IDs", + mode: { + ignore: "Ignoriere IDs der CSV-Datei und erstelle neue", + update: "Aktualisiere existierende Benutzer", + }, + }, + passwords: { + header: "Passwörter", + all_passwords_present: "Passwörter in jedem Eintrag vorhanden", + count_passwords_present: + "%{smart_count} Eintrag mit Passwort |||| %{smart_count} Einträge mit Passwörtern", + use_passwords: "Verwende Passwörter aus der CSV Datei", + }, + upload: { + header: "CSV Datei importieren", + explanation: + "Hier können Sie eine Datei mit kommagetrennten Daten hochladen, die verwendet werden um Benutzer anzulegen oder zu ändern. Die Datei muss mindestens die Felder 'id' und 'displayname' enthalten. Hier können Sie eine Beispieldatei herunterladen und anpassen: ", + }, + startImport: { + simulate_only: "Nur simulieren", + run_import: "Importieren", + }, + results: { + header: "Ergebnis", + total: + "%{smart_count} Eintrag insgesamt |||| %{smart_count} Einträge insgesamt", + successful: "%{smart_count} Einträge erfolgreich importiert", + skipped: "%{smart_count} Einträge übersprungen", + download_skipped: "Übersprungene Einträge herunterladen", + with_error: + "%{smart_count} Eintrag mit Fehlern ||| %{smart_count} Einträge mit Fehlern", + simulated_only: "Import-Vorgang war nur simuliert", + }, + }, + }, resources: { users: { backtolist: "Zurück zur Liste", diff --git a/src/i18n/en.js b/src/i18n/en.js index 7e01611..997fdf6 100644 --- a/src/i18n/en.js +++ b/src/i18n/en.js @@ -27,6 +27,74 @@ export default { }, }, }, + import_users: { + error: { + at_entry: "At entry %{entry}: %{message}", + error: "Error", + required_field: "Required field '%{field}' is not present", + invalid_value: + "Invalid value on line %{row}. '%{field}' field may only be 'true' or 'false'", + unreasonably_big: + "Refused to load unreasonably big file of %{size} megabytes", + already_in_progress: "An import run is already in progress", + id_exits: "ID %{id} already present", + }, + title: "Import users via CSV", + goToPdf: "Go to PDF", + cards: { + importstats: { + header: "Import users", + users_total: + "%{smart_count} user in CSV file |||| %{smart_count} users in CSV file", + guest_count: "%{smart_count} guest |||| %{smart_count} guests", + admin_count: "%{smart_count} admin |||| %{smart_count} admins", + }, + conflicts: { + header: "Conflict strategy", + mode: { + stop: "Stop on conflict", + skip: "Show error and skip on conflict", + }, + }, + ids: { + header: "IDs", + all_ids_present: "IDs present on every entry", + count_ids_present: + "%{smart_count} entry with ID |||| %{smart_count} entries with IDs", + mode: { + ignore: "Ignore IDs in CSV and create new ones", + update: "Update existing records", + }, + }, + passwords: { + header: "Passwords", + all_passwords_present: "Passwords present on every entry", + count_passwords_present: + "%{smart_count} entry with password |||| %{smart_count} entries with passwords", + use_passwords: "Use passwords from CSV", + }, + upload: { + header: "Input CSV file", + explanation: + "Here you can upload a file with comma separated values that is processed to create or update users. The file must include the fields 'id' and 'displayname'. You can download and adapt an example file here: ", + }, + startImport: { + simulate_only: "Simulate only", + run_import: "Import", + }, + results: { + header: "Import results", + total: + "%{smart_count} entry in total |||| %{smart_count} entries in total", + successful: "%{smart_count} entries successfully imported", + skipped: "%{smart_count} entries skipped", + download_skipped: "Download skipped records", + with_error: + "%{smart_count} entry with errors ||| %{smart_count} entries with errors", + simulated_only: "Run was only simulated", + }, + }, + }, resources: { users: { backtolist: "Back to list",