FGFS-4.1/phpvms
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix rowmapper generator, check for fields, map users #443 (#583)

Browse Source 
* Fix rowmapper generator, check for fields, map users #443

* Formatting

* Remove value store at the end of the import

* Update the notes for the importer about users

* Uncomment importers disabled during testing
This commit is contained in:
Nabeel S 2020-02-23 19:48:28 -05:00 committed by GitHub
parent a1d6fa17ad
commit dfbaa1afd3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 214 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
app/Services/UserService.php
View File

@ -135,6 +135,13 @@ class UserService extends Service
return $user;
}
/**
* Return true or false if a pilot ID already exists
*
* @param int $pilot_id
*
* @return bool
*/
public function isPilotIdAlreadyUsed(int $pilot_id): bool
{
return User::where('pilot_id', '=', $pilot_id)->exists();

2
modules/Importer/Resources/views/step1-configure.blade.php
View File

@ -10,6 +10,8 @@
<td colspan="2">
<h4>IMPORTANT NOTES</h4>
<ul>
<li>The first user's password (admin) will be "admin". Please change it after logging in</li>
<li>User passwords will be reset and they will need to use "Forgot Password" to reset it</li>
<li>If you have more than 1000 PIREPs or flights, it's best to use the command-line importer!
<a href="http://docs.phpvms.net/setup/importing-from-v2-v5" target="_blank">Click here</a> to
see the documentation of how to use it.

12
modules/Importer/Services/BaseImporter.php
View File

@ -8,6 +8,7 @@ use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Support\Facades\Log;
use Modules\Importer\Utils\IdMapper;
use Modules\Importer\Utils\ImporterDB;
@ -27,7 +28,7 @@ abstract class BaseImporter implements ShouldQueue
/**
* The mapper class used for old IDs to new IDs
*
* @var \Illuminate\Contracts\Foundation\Application|mixed
* @var IdMapper
*/
protected $idMapper;
@ -38,6 +39,13 @@ abstract class BaseImporter implements ShouldQueue
*/
protected $table;
/**
* The column used for the ID, used for the ORDER BY
*
* @var string
*/
protected $idField = 'id';
public function __construct()
{
$importerService = app(ImporterService::class);
@ -66,6 +74,8 @@ abstract class BaseImporter implements ShouldQueue
$start = 0;
$total_rows = $this->db->getTotalRows($this->table);
Log::info('Found '.$total_rows.' rows for '.$this->table);
do {
$end = $start + $this->db->batchSize;
if ($end > $total_rows) {

8
modules/Importer/Services/ImporterService.php
View File

@ -6,6 +6,7 @@ use App\Contracts\Service;
use App\Repositories\KvpRepository;
use Exception;
use Illuminate\Http\Request;
use Illuminate\Support\Facades\Log;
use Modules\Importer\Services\Importers\AircraftImporter;
use Modules\Importer\Services\Importers\AirlineImporter;
use Modules\Importer\Services\Importers\AirportImporter;
@ -130,6 +131,11 @@ class ImporterService extends Service
/** @var $importerInst \Modules\Importer\Services\BaseImporter */
$importerInst = new $importer();
$importerInst->run($start);
try {
$importerInst->run($start);
} catch (Exception $e) {
Log::error('Error running importer: '.$e->getMessage());
}
}
}

5
modules/Importer/Services/Importers/AircraftImporter.php
View File

@ -9,7 +9,7 @@ use Modules\Importer\Services\BaseImporter;
class AircraftImporter extends BaseImporter
{
public $table = 'aircraft';
protected $table = 'aircraft';
/**
* CONSTANTS
@ -25,7 +25,8 @@ class AircraftImporter extends BaseImporter
$this->info('Subfleet ID is '.$subfleet->id);
$count = 0;
foreach ($this->db->readRows($this->table, $start) as $row) {
$rows = $this->db->readRows($this->table, $this->idField, $start);
foreach ($rows as $row) {
$where = [
'name' => $row->fullname,
'registration' => $row->registration,

3
modules/Importer/Services/Importers/AirlineImporter.php
View File

@ -18,7 +18,8 @@ class AirlineImporter extends BaseImporter
$this->comment('--- AIRLINE IMPORT ---');
$count = 0;
foreach ($this->db->readRows($this->table, $start) as $row) {
$rows = $this->db->readRows($this->table, $this->idField, $start);
foreach ($rows as $row) {
$attrs = [
'iata' => $row->code,
'name' => $row->name,

3
modules/Importer/Services/Importers/AirportImporter.php
View File

@ -25,7 +25,8 @@ class AirportImporter extends BaseImporter
];
$count = 0;
foreach ($this->db->readRows($this->table, $start, $fields) as $row) {
$rows = $this->db->readRows($this->table, $this->idField, $start, $fields);
foreach ($rows as $row) {
$attrs = [
'id' => trim($row->icao),
'icao' => trim($row->icao),

18
modules/Importer/Services/Importers/ClearDatabase.php
View File

@ -3,6 +3,7 @@
namespace Modules\Importer\Services\Importers;
use App\Models\Acars;
use App\Models\Aircraft;
use App\Models\Airline;
use App\Models\Airport;
use App\Models\Bid;
@ -15,6 +16,7 @@ use App\Models\Journal;
use App\Models\JournalTransaction;
use App\Models\News;
use App\Models\Pirep;
use App\Models\Role;
use App\Models\Subfleet;
use App\Models\User;
use App\Models\UserAward;
@ -68,12 +70,7 @@ class ClearDatabase extends BaseImporter
FlightFieldValue::truncate();
Flight::truncate();
Subfleet::truncate();
// Clear permissions
// DB::table('permission_role')->truncate();
// DB::table('permission_user')->truncate();
// DB::table('role_user')->truncate();
// Role::truncate();
Aircraft::truncate();
Airline::truncate();
Airport::truncate();
@ -83,6 +80,15 @@ class ClearDatabase extends BaseImporter
UserAward::truncate();
User::truncate();
// Clear permissions
DB::table('permission_role')->truncate();
DB::table('permission_user')->truncate();
DB::table('role_user')->truncate();
// Role::truncate();
DB::statement('SET FOREIGN_KEY_CHECKS=1');
$this->idMapper->clear();
}
}

10
modules/Importer/Services/Importers/FinalizeImporter.php
View File

@ -34,6 +34,7 @@ class FinalizeImporter extends BaseImporter
{
$this->findLastPireps();
$this->recalculateUserStats();
$this->clearValueStore();
}
/**
@ -41,6 +42,7 @@ class FinalizeImporter extends BaseImporter
*/
protected function findLastPireps()
{
// TODO
}
/**
@ -55,4 +57,12 @@ class FinalizeImporter extends BaseImporter
$userSvc->recalculateStats($user);
});
}
/**
* Clear the value store of any old value mappings
*/
protected function clearValueStore()
{
$this->idMapper->clear();
}
}

5
modules/Importer/Services/Importers/FlightImporter.php
View File

@ -24,13 +24,14 @@ class FlightImporter extends BaseImporter
'flightlevel',
'deptime',
'arrtime',
'flightttime',
'flighttime',
'notes',
'enabled',
];
$count = 0;
foreach ($this->db->readRows($this->table, $start, $fields) as $row) {
$rows = $this->db->readRows($this->table, $this->idField, $start, $fields);
foreach ($rows as $row) {
$airline_id = $this->idMapper->getMapping('airlines', $row->code);
$flight_num = trim($row->flightnum);

11
modules/Importer/Services/Importers/GroupImporter.php
View File

@ -12,6 +12,7 @@ use Modules\Importer\Services\BaseImporter;
class GroupImporter extends BaseImporter
{
protected $table = 'groups';
protected $idField = 'groupid';
/**
* Permissions in the legacy system, mapping them to the current system
@ -68,7 +69,15 @@ class GroupImporter extends BaseImporter
$roleSvc = app(RoleService::class);
$count = 0;
foreach ($this->db->readRows($this->table, $start) as $row) {
$rows = $this->db->readRows($this->table, $this->idField, $start);
foreach ($rows as $row) {
// Legacy "administrator" role is now "admin", just map that 1:1
if (strtolower($row->name) === 'administrators') {
$role = Role::where('name', 'admin')->first();
$this->idMapper->addMapping('group', $row->groupid, $role->id);
continue;
}
$name = str_slug($row->name);
$role = Role::firstOrCreate(
['name' => $name],

11
modules/Importer/Services/Importers/PirepImporter.php
View File

@ -11,6 +11,7 @@ use Modules\Importer\Services\BaseImporter;
class PirepImporter extends BaseImporter
{
protected $table = 'pireps';
protected $idField = 'pirepid';
public function run($start = 0)
{
@ -32,11 +33,17 @@ class PirepImporter extends BaseImporter
'distance',
'flighttime_stamp',
'flighttype',
'flightlevel',
];
// See if there's a flightlevel column, export that if there is
$columns = $this->db->getColumns($this->table);
if (in_array('flightlevel', $columns)) {
$fields[] = 'flightlevel';
}
$count = 0;
foreach ($this->db->readRows($this->table, $start, $fields) as $row) {
$rows = $this->db->readRows($this->table, $this->idField, $start, $fields);
foreach ($rows as $row) {
$pirep_id = $row->pirepid;
$user_id = $this->idMapper->getMapping('users', $row->pilotid);
$airline_id = $this->idMapper->getMapping('airlines', $row->code);

4
modules/Importer/Services/Importers/RankImport.php
View File

@ -8,13 +8,15 @@ use Modules\Importer\Services\BaseImporter;
class RankImport extends BaseImporter
{
protected $table = 'ranks';
protected $idField = 'rankid';
public function run($start = 0)
{
$this->comment('--- RANK IMPORT ---');
$count = 0;
foreach ($this->db->readRows($this->table, $start) as $row) {
$rows = $this->db->readRows($this->table, $this->idField, $start);
foreach ($rows as $row) {
$rank = Rank::firstOrCreate(['name' => $row->rank], [
'image_url' => $row->rankimage,
'hours' => $row->minhours,

35
modules/Importer/Services/Importers/UserImport.php
View File

@ -3,6 +3,7 @@
namespace Modules\Importer\Services\Importers;
use App\Models\Enums\UserState;
use App\Models\Role;
use App\Models\User;
use App\Services\UserService;
use App\Support\Units\Time;
@ -15,6 +16,7 @@ use Modules\Importer\Services\BaseImporter;
class UserImport extends BaseImporter
{
protected $table = 'pilots';
protected $idField = 'pilotid';
/**
* @var UserService
@ -29,7 +31,8 @@ class UserImport extends BaseImporter
$count = 0;
$first_row = true;
foreach ($this->db->readRows($this->table, $start) as $row) {
$rows = $this->db->readRows($this->table, $this->idField, $start);
foreach ($rows as $row) {
$pilot_id = $row->pilotid; // This isn't their actual ID
$name = $row->firstname.' '.$row->lastname;
@ -49,8 +52,13 @@ class UserImport extends BaseImporter
// Look for a user with that pilot ID already. If someone has it
if ($this->userSvc->isPilotIdAlreadyUsed($pilot_id)) {
Log::info('User with pilot id '.$pilot_id.' exists, reassigning');
$pilot_id = $this->userSvc->getNextAvailablePilotId();
Log::info('User with pilot id '.$pilot_id.' exists');
// Is this the same user? If not, get a new pilot ID
$user_exist = User::where('pilot_id', $pilot_id)->first();
if ($user_exist->email !== $row->email) {
$pilot_id = $this->userSvc->getNextAvailablePilotId();
}
}
$attrs = [
@ -91,9 +99,26 @@ class UserImport extends BaseImporter
{
// Be default add them to the user role, and then determine if they
// belong to any other groups, and add them to that
$this->userSvc->addUserToRole($user, 'user');
$roleMappings = [];
$newRoles = [];
// Figure out what other groups they belong to
// Figure out what other groups they belong to... read from the old table, and map
// them to the new group(s)
$old_user_groups = $this->db->findBy('groupmembers', ['pilotid' => $old_pilot_id]);
foreach ($old_user_groups as $oldGroup) {
$newRoleId = $this->idMapper->getMapping('group', $oldGroup->groupid);
// Only lookup a new role ID if found
// if (!in_array($newRoleId, $roleMappings)) {
// $roleMappings[$newRoleId] = Role::where(['id' => $newRoleId])->first();
// }
// $newRoles[] = $roleMappings[$newRoleId];
$newRoles[] = $newRoleId;
}
// Assign the groups to the new user
$user->attachRoles($newRoles);
}
/**

8
modules/Importer/Utils/IdMapper.php
View File

@ -46,4 +46,12 @@ class IdMapper extends Service
return $this->valueStore->get($key_name);
}
/**
* Clear the value store
*/
public function clear()
{
$this->valueStore->flush();
}
}

117
modules/Importer/Utils/ImporterDB.php
View File

@ -85,6 +85,28 @@ class ImporterDB
return $table;
}
/**
* Get the names of the columns for a particular table
*
* @param $table
*
* @return mixed
*/
public function getColumns($table)
{
$this->connect();
$sql = 'SHOW COLUMNS FROM '.$this->tableName($table);
$result = $this->conn->query($sql)->fetchAll();
$rows = [];
foreach ($result as $row) {
$rows[] = $row->Field;
}
return $rows;
}
/**
* @param $table
*
@ -102,50 +124,95 @@ class ImporterDB
return (int) $rows;
}
/**
* Read rows from a table with a given assoc array. Simple
*
* @param string $table
* @param array $attrs
*
* @return false|\PDOStatement
*/
public function findBy($table, array $attrs)
{
$this->connect();
$where = [];
foreach ($attrs as $col => $value) {
$where[] = $col.'=\''.$value.'\'';
}
$where = implode(' AND ', $where);
$sql = implode(' ', [
'SELECT',
'*',
'FROM',
$this->tableName($table),
'WHERE',
$where,
]);
return $this->conn->query($sql);
}
/**
* Read all the rows in a table, but read them in a batched manner
*
* @param string $table The name of the table
* @param int [$start_offset]
* @param string [$fields]
* @param string $table The name of the table
* @param string $order_by Column to order by
* @param int $start_offset
* @param string $fields
*
* @return \Generator
* @return array
*/
public function readRows($table, $start_offset = 0, $fields = '*')
public function readRows($table, $order_by = 'id', $start_offset = 0, $fields = '*')
{
$this->connect();
$offset = $start_offset;
$total_rows = $this->getTotalRows($table);
// $total_rows = $this->getTotalRows($table);
while ($offset < $total_rows) {
$rows_to_read = $offset + $this->batchSize;
if ($rows_to_read > $total_rows) {
$rows_to_read = $total_rows;
}
// Log::info('Reading '.$offset.' to '.$rows_to_read.' of '.$total_rows);
yield from $this->readRowsOffset($table, $this->batchSize, $offset, $fields);
$offset += $this->batchSize;
$rows = [];
$result = $this->readRowsOffset($table, $this->batchSize, $offset, $order_by, $fields);
if ($result === false) {
return [];
}
try {
foreach ($result as $row) {
$rows[] = $row;
}
} catch (Exception $e) {
Log::error('foreach rows error: '.$e->getMessage());
}
return $rows;
}
/**
* @param string $table
* @param int $limit Number of rows to read
* @param int $offset Where to start from
* @param string [$fields]
* @param int $limit Number of rows to read
* @param int $offset Where to start from
* @param $order_by
* @param string $fields
*
* @return \Generator
* @return false|\PDOStatement|void
*/
public function readRowsOffset($table, $limit, $offset, $fields = '*')
public function readRowsOffset($table, $limit, $offset, $order_by, $fields = '*')
{
if (is_array($fields)) {
$fields = implode(',', $fields);
}
$sql = 'SELECT '.$fields.' FROM '.$this->tableName($table).' LIMIT '.$limit.' OFFSET '.$offset;
$sql = implode(' ', [
'SELECT',
$fields,
'FROM',
$this->tableName($table),
'ORDER BY '.$order_by.' ASC',
'LIMIT '.$limit,
'OFFSET '.$offset,
]);
try {
$result = $this->conn->query($sql);
@ -153,9 +220,7 @@ class ImporterDB
return;
}
foreach ($result as $row) {
yield $row;
}
return $result;
} catch (PDOException $e) {
// Without incrementing the offset, it should re-run the same query
Log::error('Error readRowsOffset: '.$e->getMessage());
@ -163,6 +228,8 @@ class ImporterDB
if (strpos($e->getMessage(), 'server has gone away') !== false) {
$this->connect();
}
} catch (\Exception $e) {
Log::error('Error readRowsOffset: '.$e->getMessage());
}
}
}

4
resources/views/system/acars/config.blade.php
View File

@ -1,4 +1,6 @@
<?xml version = "1.0" encoding = "utf-8"?>
@php
echo '<?xml version = "1.0" encoding = "utf-8"?>';
@endphp
<AcarsConfiguration xmlns:i="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://schemas.phpvms.net/acars/config">
<AirlineUrl>{{ config('app.url') }}</AirlineUrl>
<ApiKey>{{ $user->api_key }}</ApiKey>