Fix rowmapper generator, check for fields, map users #443 (#583)

* Fix rowmapper generator, check for fields, map users #443

* Formatting

* Remove value store at the end of the import

* Update the notes for the importer about users

* Uncomment importers disabled during testing

app/Services/UserService.php

@@ -135,6 +135,13 @@ class UserService extends Service
         return $user;
     }
 
+    /**
+     * Return true or false if a pilot ID already exists
+     *
+     * @param int $pilot_id
+     *
+     * @return bool
+     */
     public function isPilotIdAlreadyUsed(int $pilot_id): bool
     {
         return User::where('pilot_id', '=', $pilot_id)->exists();

modules/Importer/Resources/views/step1-configure.blade.php

@@ -10,6 +10,8 @@
         <td colspan="2">
           <h4>IMPORTANT NOTES</h4>
           <ul>
+            <li>The first user's password (admin) will be "admin". Please change it after logging in</li>
+            <li>User passwords will be reset and they will need to use "Forgot Password" to reset it</li>
             <li>If you have more than 1000 PIREPs or flights, it's best to use the command-line importer!
               <a href="http://docs.phpvms.net/setup/importing-from-v2-v5" target="_blank">Click here</a> to
               see the documentation of how to use it.

modules/Importer/Services/BaseImporter.php

@@ -8,6 +8,7 @@ use Illuminate\Bus\Queueable;
 use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Support\Facades\Log;
 use Modules\Importer\Utils\IdMapper;
 use Modules\Importer\Utils\ImporterDB;
 
@@ -27,7 +28,7 @@ abstract class BaseImporter implements ShouldQueue
     /**
      * The mapper class used for old IDs to new IDs
      *
-     * @var \Illuminate\Contracts\Foundation\Application|mixed
+     * @var IdMapper
      */
     protected $idMapper;
 
@@ -38,6 +39,13 @@ abstract class BaseImporter implements ShouldQueue
      */
     protected $table;
 
+    /**
+     * The column used for the ID, used for the ORDER BY
+     *
+     * @var string
+     */
+    protected $idField = 'id';
+
     public function __construct()
     {
         $importerService = app(ImporterService::class);
@@ -66,6 +74,8 @@ abstract class BaseImporter implements ShouldQueue
 
         $start = 0;
         $total_rows = $this->db->getTotalRows($this->table);
+        Log::info('Found '.$total_rows.' rows for '.$this->table);
+
         do {
             $end = $start + $this->db->batchSize;
             if ($end > $total_rows) {

modules/Importer/Services/ImporterService.php

@@ -6,6 +6,7 @@ use App\Contracts\Service;
 use App\Repositories\KvpRepository;
 use Exception;
 use Illuminate\Http\Request;
+use Illuminate\Support\Facades\Log;
 use Modules\Importer\Services\Importers\AircraftImporter;
 use Modules\Importer\Services\Importers\AirlineImporter;
 use Modules\Importer\Services\Importers\AirportImporter;
@@ -130,6 +131,11 @@ class ImporterService extends Service
 
         /** @var $importerInst \Modules\Importer\Services\BaseImporter */
         $importerInst = new $importer();
-        $importerInst->run($start);
+
+        try {
+            $importerInst->run($start);
+        } catch (Exception $e) {
+            Log::error('Error running importer: '.$e->getMessage());
+        }
     }
 }

modules/Importer/Services/Importers/AircraftImporter.php

@@ -9,7 +9,7 @@ use Modules\Importer\Services\BaseImporter;
 
 class AircraftImporter extends BaseImporter
 {
-    public $table = 'aircraft';
+    protected $table = 'aircraft';
 
     /**
      * CONSTANTS
@@ -25,7 +25,8 @@ class AircraftImporter extends BaseImporter
         $this->info('Subfleet ID is '.$subfleet->id);
 
         $count = 0;
-        foreach ($this->db->readRows($this->table, $start) as $row) {
+        $rows = $this->db->readRows($this->table, $this->idField, $start);
+        foreach ($rows as $row) {
             $where = [
                 'name'         => $row->fullname,
                 'registration' => $row->registration,

modules/Importer/Services/Importers/AirlineImporter.php

@@ -18,7 +18,8 @@ class AirlineImporter extends BaseImporter
         $this->comment('--- AIRLINE IMPORT ---');
 
         $count = 0;
-        foreach ($this->db->readRows($this->table, $start) as $row) {
+        $rows = $this->db->readRows($this->table, $this->idField, $start);
+        foreach ($rows as $row) {
             $attrs = [
                 'iata'   => $row->code,
                 'name'   => $row->name,

modules/Importer/Services/Importers/AirportImporter.php

@@ -25,7 +25,8 @@ class AirportImporter extends BaseImporter
         ];
 
         $count = 0;
-        foreach ($this->db->readRows($this->table, $start, $fields) as $row) {
+        $rows = $this->db->readRows($this->table, $this->idField, $start, $fields);
+        foreach ($rows as $row) {
             $attrs = [
                 'id'      => trim($row->icao),
                 'icao'    => trim($row->icao),

modules/Importer/Services/Importers/ClearDatabase.php

@@ -3,6 +3,7 @@
 namespace Modules\Importer\Services\Importers;
 
 use App\Models\Acars;
+use App\Models\Aircraft;
 use App\Models\Airline;
 use App\Models\Airport;
 use App\Models\Bid;
@@ -15,6 +16,7 @@ use App\Models\Journal;
 use App\Models\JournalTransaction;
 use App\Models\News;
 use App\Models\Pirep;
+use App\Models\Role;
 use App\Models\Subfleet;
 use App\Models\User;
 use App\Models\UserAward;
@@ -68,12 +70,7 @@ class ClearDatabase extends BaseImporter
         FlightFieldValue::truncate();
         Flight::truncate();
         Subfleet::truncate();
-
+        Aircraft::truncate();
-        // Clear permissions
-        //        DB::table('permission_role')->truncate();
-        //        DB::table('permission_user')->truncate();
-        //        DB::table('role_user')->truncate();
-        //        Role::truncate();
 
         Airline::truncate();
         Airport::truncate();
@@ -83,6 +80,15 @@ class ClearDatabase extends BaseImporter
         UserAward::truncate();
         User::truncate();
 
+        // Clear permissions
+        DB::table('permission_role')->truncate();
+        DB::table('permission_user')->truncate();
+        DB::table('role_user')->truncate();
+
+        // Role::truncate();
+
         DB::statement('SET FOREIGN_KEY_CHECKS=1');
+
+        $this->idMapper->clear();
     }
 }

modules/Importer/Services/Importers/FinalizeImporter.php

@@ -34,6 +34,7 @@ class FinalizeImporter extends BaseImporter
     {
         $this->findLastPireps();
         $this->recalculateUserStats();
+        $this->clearValueStore();
     }
 
     /**
@@ -41,6 +42,7 @@ class FinalizeImporter extends BaseImporter
      */
     protected function findLastPireps()
     {
+        // TODO
     }
 
     /**
@@ -55,4 +57,12 @@ class FinalizeImporter extends BaseImporter
             $userSvc->recalculateStats($user);
         });
     }
+
+    /**
+     * Clear the value store of any old value mappings
+     */
+    protected function clearValueStore()
+    {
+        $this->idMapper->clear();
+    }
 }

modules/Importer/Services/Importers/FlightImporter.php

@@ -24,13 +24,14 @@ class FlightImporter extends BaseImporter
             'flightlevel',
             'deptime',
             'arrtime',
-            'flightttime',
+            'flighttime',
             'notes',
             'enabled',
         ];
 
         $count = 0;
-        foreach ($this->db->readRows($this->table, $start, $fields) as $row) {
+        $rows = $this->db->readRows($this->table, $this->idField, $start, $fields);
+        foreach ($rows as $row) {
             $airline_id = $this->idMapper->getMapping('airlines', $row->code);
 
             $flight_num = trim($row->flightnum);

modules/Importer/Services/Importers/GroupImporter.php

@@ -12,6 +12,7 @@ use Modules\Importer\Services\BaseImporter;
 class GroupImporter extends BaseImporter
 {
     protected $table = 'groups';
+    protected $idField = 'groupid';
 
     /**
      * Permissions in the legacy system, mapping them to the current system
@@ -68,7 +69,15 @@ class GroupImporter extends BaseImporter
         $roleSvc = app(RoleService::class);
 
         $count = 0;
-        foreach ($this->db->readRows($this->table, $start) as $row) {
+        $rows = $this->db->readRows($this->table, $this->idField, $start);
+        foreach ($rows as $row) {
+            // Legacy "administrator" role is now "admin", just map that 1:1
+            if (strtolower($row->name) === 'administrators') {
+                $role = Role::where('name', 'admin')->first();
+                $this->idMapper->addMapping('group', $row->groupid, $role->id);
+                continue;
+            }
+
             $name = str_slug($row->name);
             $role = Role::firstOrCreate(
                 ['name' => $name],

modules/Importer/Services/Importers/PirepImporter.php

@@ -11,6 +11,7 @@ use Modules\Importer\Services\BaseImporter;
 class PirepImporter extends BaseImporter
 {
     protected $table = 'pireps';
+    protected $idField = 'pirepid';
 
     public function run($start = 0)
     {
@@ -32,11 +33,17 @@ class PirepImporter extends BaseImporter
             'distance',
             'flighttime_stamp',
             'flighttype',
-            'flightlevel',
         ];
 
+        // See if there's a flightlevel column, export that if there is
+        $columns = $this->db->getColumns($this->table);
+        if (in_array('flightlevel', $columns)) {
+            $fields[] = 'flightlevel';
+        }
+
         $count = 0;
-        foreach ($this->db->readRows($this->table, $start, $fields) as $row) {
+        $rows = $this->db->readRows($this->table, $this->idField, $start, $fields);
+        foreach ($rows as $row) {
             $pirep_id = $row->pirepid;
             $user_id = $this->idMapper->getMapping('users', $row->pilotid);
             $airline_id = $this->idMapper->getMapping('airlines', $row->code);

modules/Importer/Services/Importers/RankImport.php

@@ -8,13 +8,15 @@ use Modules\Importer\Services\BaseImporter;
 class RankImport extends BaseImporter
 {
     protected $table = 'ranks';
+    protected $idField = 'rankid';
 
     public function run($start = 0)
     {
         $this->comment('--- RANK IMPORT ---');
 
         $count = 0;
-        foreach ($this->db->readRows($this->table, $start) as $row) {
+        $rows = $this->db->readRows($this->table, $this->idField, $start);
+        foreach ($rows as $row) {
             $rank = Rank::firstOrCreate(['name' => $row->rank], [
                 'image_url' => $row->rankimage,
                 'hours'     => $row->minhours,

modules/Importer/Services/Importers/UserImport.php

@@ -3,6 +3,7 @@
 namespace Modules\Importer\Services\Importers;
 
 use App\Models\Enums\UserState;
+use App\Models\Role;
 use App\Models\User;
 use App\Services\UserService;
 use App\Support\Units\Time;
@@ -15,6 +16,7 @@ use Modules\Importer\Services\BaseImporter;
 class UserImport extends BaseImporter
 {
     protected $table = 'pilots';
+    protected $idField = 'pilotid';
 
     /**
      * @var UserService
@@ -29,7 +31,8 @@ class UserImport extends BaseImporter
 
         $count = 0;
         $first_row = true;
-        foreach ($this->db->readRows($this->table, $start) as $row) {
+        $rows = $this->db->readRows($this->table, $this->idField, $start);
+        foreach ($rows as $row) {
             $pilot_id = $row->pilotid; // This isn't their actual ID
             $name = $row->firstname.' '.$row->lastname;
 
@@ -49,8 +52,13 @@ class UserImport extends BaseImporter
 
             // Look for a user with that pilot ID already. If someone has it
             if ($this->userSvc->isPilotIdAlreadyUsed($pilot_id)) {
-                Log::info('User with pilot id '.$pilot_id.' exists, reassigning');
+                Log::info('User with pilot id '.$pilot_id.' exists');
-                $pilot_id = $this->userSvc->getNextAvailablePilotId();
+
+                // Is this the same user? If not, get a new pilot ID
+                $user_exist = User::where('pilot_id', $pilot_id)->first();
+                if ($user_exist->email !== $row->email) {
+                    $pilot_id = $this->userSvc->getNextAvailablePilotId();
+                }
             }
 
             $attrs = [
@@ -91,9 +99,26 @@ class UserImport extends BaseImporter
     {
         // Be default add them to the user role, and then determine if they
         // belong to any other groups, and add them to that
-        $this->userSvc->addUserToRole($user, 'user');
+        $roleMappings = [];
+        $newRoles = [];
 
-        // Figure out what other groups they belong to
+        // Figure out what other groups they belong to... read from the old table, and map
+        // them to the new group(s)
+        $old_user_groups = $this->db->findBy('groupmembers', ['pilotid' => $old_pilot_id]);
+        foreach ($old_user_groups as $oldGroup) {
+            $newRoleId = $this->idMapper->getMapping('group', $oldGroup->groupid);
+
+            // Only lookup a new role ID if found
+            // if (!in_array($newRoleId, $roleMappings)) {
+            //     $roleMappings[$newRoleId] = Role::where(['id' => $newRoleId])->first();
+            // }
+
+            // $newRoles[] = $roleMappings[$newRoleId];
+            $newRoles[] = $newRoleId;
+        }
+
+        // Assign the groups to the new user
+        $user->attachRoles($newRoles);
     }
 
     /**

modules/Importer/Utils/IdMapper.php

@@ -46,4 +46,12 @@ class IdMapper extends Service
 
         return $this->valueStore->get($key_name);
     }
+
+    /**
+     * Clear the value store
+     */
+    public function clear()
+    {
+        $this->valueStore->flush();
+    }
 }

modules/Importer/Utils/ImporterDB.php

@@ -85,6 +85,28 @@ class ImporterDB
         return $table;
     }
 
+    /**
+     * Get the names of the columns for a particular table
+     *
+     * @param $table
+     *
+     * @return mixed
+     */
+    public function getColumns($table)
+    {
+        $this->connect();
+
+        $sql = 'SHOW COLUMNS FROM '.$this->tableName($table);
+        $result = $this->conn->query($sql)->fetchAll();
+
+        $rows = [];
+        foreach ($result as $row) {
+            $rows[] = $row->Field;
+        }
+
+        return $rows;
+    }
+
     /**
      * @param $table
      *
@@ -102,50 +124,95 @@ class ImporterDB
         return (int) $rows;
     }
 
+    /**
+     * Read rows from a table with a given assoc array. Simple
+     *
+     * @param string $table
+     * @param array  $attrs
+     *
+     * @return false|\PDOStatement
+     */
+    public function findBy($table, array $attrs)
+    {
+        $this->connect();
+
+        $where = [];
+        foreach ($attrs as $col => $value) {
+            $where[] = $col.'=\''.$value.'\'';
+        }
+
+        $where = implode(' AND ', $where);
+
+        $sql = implode(' ', [
+            'SELECT',
+            '*',
+            'FROM',
+            $this->tableName($table),
+            'WHERE',
+            $where,
+        ]);
+
+        return $this->conn->query($sql);
+    }
+
     /**
      * Read all the rows in a table, but read them in a batched manner
      *
-     * @param string $table The name of the table
+     * @param string $table        The name of the table
-     * @param int    [$start_offset]
+     * @param string $order_by     Column to order by
-     * @param string [$fields]
+     * @param int    $start_offset
+     * @param string $fields
      *
-     * @return \Generator
+     * @return array
      */
-    public function readRows($table, $start_offset = 0, $fields = '*')
+    public function readRows($table, $order_by = 'id', $start_offset = 0, $fields = '*')
     {
         $this->connect();
 
         $offset = $start_offset;
-        $total_rows = $this->getTotalRows($table);
+        // $total_rows = $this->getTotalRows($table);
 
-        while ($offset < $total_rows) {
+        $rows = [];
-            $rows_to_read = $offset + $this->batchSize;
+        $result = $this->readRowsOffset($table, $this->batchSize, $offset, $order_by, $fields);
-            if ($rows_to_read > $total_rows) {
+        if ($result === false) {
-                $rows_to_read = $total_rows;
+            return [];
-            }
-
-            // Log::info('Reading '.$offset.' to '.$rows_to_read.' of '.$total_rows);
-            yield from $this->readRowsOffset($table, $this->batchSize, $offset, $fields);
-
-            $offset += $this->batchSize;
         }
+
+        try {
+            foreach ($result as $row) {
+                $rows[] = $row;
+            }
+        } catch (Exception $e) {
+            Log::error('foreach rows error: '.$e->getMessage());
+        }
+
+        return $rows;
     }
 
     /**
      * @param string $table
-     * @param int    $limit  Number of rows to read
+     * @param int    $limit    Number of rows to read
-     * @param int    $offset Where to start from
+     * @param int    $offset   Where to start from
-     * @param string [$fields]
+     * @param        $order_by
+     * @param string $fields
      *
-     * @return \Generator
+     * @return false|\PDOStatement|void
      */
-    public function readRowsOffset($table, $limit, $offset, $fields = '*')
+    public function readRowsOffset($table, $limit, $offset, $order_by, $fields = '*')
     {
         if (is_array($fields)) {
             $fields = implode(',', $fields);
         }
 
-        $sql = 'SELECT '.$fields.' FROM '.$this->tableName($table).' LIMIT '.$limit.' OFFSET '.$offset;
+        $sql = implode(' ', [
+            'SELECT',
+            $fields,
+            'FROM',
+            $this->tableName($table),
+            'ORDER BY '.$order_by.' ASC',
+            'LIMIT '.$limit,
+            'OFFSET '.$offset,
+        ]);
 
         try {
             $result = $this->conn->query($sql);
@@ -153,9 +220,7 @@ class ImporterDB
                 return;
             }
 
-            foreach ($result as $row) {
+            return $result;
-                yield $row;
-            }
         } catch (PDOException $e) {
             // Without incrementing the offset, it should re-run the same query
             Log::error('Error readRowsOffset: '.$e->getMessage());
@@ -163,6 +228,8 @@ class ImporterDB
             if (strpos($e->getMessage(), 'server has gone away') !== false) {
                 $this->connect();
             }
+        } catch (\Exception $e) {
+            Log::error('Error readRowsOffset: '.$e->getMessage());
         }
     }
 }

resources/views/system/acars/config.blade.php

@@ -1,4 +1,6 @@
-<?xml version = "1.0" encoding = "utf-8"?>
+@php
+echo '<?xml version = "1.0" encoding = "utf-8"?>';
+@endphp
 <AcarsConfiguration xmlns:i="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://schemas.phpvms.net/acars/config">
   <AirlineUrl>{{ config('app.url') }}</AirlineUrl>
   <ApiKey>{{ $user->api_key }}</ApiKey>