diff --git a/composer.json b/composer.json index df860b83b2..fbb042fb4b 100644 --- a/composer.json +++ b/composer.json @@ -14,6 +14,7 @@ "kirschbaum-development/eloquent-power-joins": "^3.0", "laravel/cashier": "^14.2", "laravel/pint": "^1.0", + "league/csv": "^9.11", "league/flysystem-aws-s3-v3": "^3.0", "nunomaduro/larastan": "^2.2", "nunomaduro/termwind": "^1.0", diff --git a/composer.lock b/composer.lock index 85b2a6c7f7..d3f7044e0b 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "d8dcadc8aa314d779fc0a7399ba6f609", + "content-hash": "643eb9c7b76d7224ee2b054f5b6e867b", "packages": [], "packages-dev": [ { @@ -3081,6 +3081,94 @@ ], "time": "2022-12-11T20:36:23+00:00" }, + { + "name": "league/csv", + "version": "9.11.0", + "source": { + "type": "git", + "url": "https://github.com/thephpleague/csv.git", + "reference": "33149c4bea4949aa4fa3d03fb11ed28682168b39" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/thephpleague/csv/zipball/33149c4bea4949aa4fa3d03fb11ed28682168b39", + "reference": "33149c4bea4949aa4fa3d03fb11ed28682168b39", + "shasum": "" + }, + "require": { + "ext-json": "*", + "ext-mbstring": "*", + "php": "^8.1.2" + }, + "require-dev": { + "doctrine/collections": "^2.1.3", + "ext-dom": "*", + "ext-xdebug": "*", + "friendsofphp/php-cs-fixer": "^v3.22.0", + "phpbench/phpbench": "^1.2.14", + "phpstan/phpstan": "^1.10.26", + "phpstan/phpstan-deprecation-rules": "^1.1.3", + "phpstan/phpstan-phpunit": "^1.3.13", + "phpstan/phpstan-strict-rules": "^1.5.1", + "phpunit/phpunit": "^10.3.1", + "symfony/var-dumper": "^6.3.3" + }, + "suggest": { + "ext-dom": "Required to use the XMLConverter and the HTMLConverter classes", + "ext-iconv": "Needed to ease transcoding CSV using iconv stream filters" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "9.x-dev" + } + }, + "autoload": { + "files": [ + "src/functions_include.php" + ], + "psr-4": { + "League\\Csv\\": "src" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Ignace Nyamagana Butera", + "email": "nyamsprod@gmail.com", + "homepage": "https://github.com/nyamsprod/", + "role": "Developer" + } + ], + "description": "CSV data manipulation made easy in PHP", + "homepage": "https://csv.thephpleague.com", + "keywords": [ + "convert", + "csv", + "export", + "filter", + "import", + "read", + "transform", + "write" + ], + "support": { + "docs": "https://csv.thephpleague.com", + "issues": "https://github.com/thephpleague/csv/issues", + "rss": "https://github.com/thephpleague/csv/releases.atom", + "source": "https://github.com/thephpleague/csv" + }, + "funding": [ + { + "url": "https://github.com/sponsors/nyamsprod", + "type": "github" + } + ], + "time": "2023-09-23T10:09:54+00:00" + }, { "name": "league/flysystem", "version": "3.18.0", diff --git a/packages/actions/composer.json b/packages/actions/composer.json index 8a4e2ab76f..bb135c98be 100644 --- a/packages/actions/composer.json +++ b/packages/actions/composer.json @@ -16,6 +16,7 @@ "illuminate/contracts": "^10.0", "illuminate/database": "^10.0", "illuminate/support": "^10.0", + "league/csv": "^9.11", "spatie/laravel-package-tools": "^1.9" }, "autoload": { diff --git a/packages/actions/database/migrations/create_failed_import_rows_table.php b/packages/actions/database/migrations/create_failed_import_rows_table.php new file mode 100644 index 0000000000..ed3b0a4e3e --- /dev/null +++ b/packages/actions/database/migrations/create_failed_import_rows_table.php @@ -0,0 +1,22 @@ +id(); + $table->json('data'); + $table->foreignId('import_id')->constrained()->cascadeOnDelete(); + $table->text('validation_error')->nullable(); + $table->timestamps(); + }); + } +}; diff --git a/packages/actions/database/migrations/create_imports_table.php b/packages/actions/database/migrations/create_imports_table.php new file mode 100644 index 0000000000..56232cc6a1 --- /dev/null +++ b/packages/actions/database/migrations/create_imports_table.php @@ -0,0 +1,27 @@ +id(); + $table->timestamp('completed_at')->nullable(); + $table->string('file_name'); + $table->string('file_path'); + $table->string('importer'); + $table->unsignedInteger('processed_rows')->default(0); + $table->unsignedInteger('total_rows'); + $table->unsignedInteger('successful_rows')->default(0); + $table->foreignId('user_id')->constrained()->cascadeOnDelete(); + $table->timestamps(); + }); + } +}; diff --git a/packages/actions/resources/lang/en/import.php b/packages/actions/resources/lang/en/import.php new file mode 100644 index 0000000000..54074b9c3c --- /dev/null +++ b/packages/actions/resources/lang/en/import.php @@ -0,0 +1,77 @@ + 'Import :label', + + 'modal' => [ + + 'heading' => 'Import :label', + + 'form' => [ + + 'file' => [ + 'label' => 'File', + 'placeholder' => 'Upload a .csv file', + ], + + 'columns' => [ + 'label' => 'Columns', + 'placeholder' => 'Select a column', + ], + + ], + + 'actions' => [ + + 'download_example' => [ + 'label' => 'Download an example .csv file.', + ], + + 'import' => [ + 'label' => 'Start import', + ], + + ], + + ], + + 'notifications' => [ + + 'completed' => [ + + 'title' => 'Import completed', + + 'actions' => [ + + 'download_failed_rows_csv' => [ + 'label' => 'Download information about the failed row|Download information about the failed rows', + ], + + ], + + ], + + 'max_rows' => [ + 'title' => 'That file is too large to import', + 'body' => 'You may not import more than :count row at once.', + ], + + 'started' => [ + 'title' => 'Import started', + 'body' => 'Your import has begun and 1 row will be processed in the background.|Your import has begun and :count rows will be processed in the background.', + ], + + ], + + 'example_csv' => [ + 'file_name' => ':importer-example', + ], + + 'failure_csv' => [ + 'file_name' => 'import-:import_id-:csv_name-failed-rows', + 'error_header' => 'error', + 'system_error' => 'System error, please contact support.', + ], + +]; diff --git a/packages/actions/routes/web.php b/packages/actions/routes/web.php new file mode 100644 index 0000000000..a2a79a8468 --- /dev/null +++ b/packages/actions/routes/web.php @@ -0,0 +1,8 @@ +name('filament.imports.failed-rows.download') + ->middleware(['web', 'auth']); diff --git a/packages/actions/src/ActionsServiceProvider.php b/packages/actions/src/ActionsServiceProvider.php index 2e4d68109e..6677d32df2 100644 --- a/packages/actions/src/ActionsServiceProvider.php +++ b/packages/actions/src/ActionsServiceProvider.php @@ -3,6 +3,7 @@ namespace Filament\Actions; use Filament\Actions\Testing\TestsActions; +use Illuminate\Filesystem\Filesystem; use Livewire\Features\SupportTesting\Testable; use Spatie\LaravelPackageTools\Package; use Spatie\LaravelPackageTools\PackageServiceProvider; @@ -13,12 +14,53 @@ class ActionsServiceProvider extends PackageServiceProvider { $package ->name('filament-actions') + ->hasCommands($this->getCommands()) + ->hasMigrations([ + 'create_imports_table', + 'create_failed_import_rows_table', + ]) + ->hasRoute('web') ->hasTranslations() ->hasViews(); } public function packageBooted(): void { + if ($this->app->runningInConsole()) { + foreach (app(Filesystem::class)->files(__DIR__ . '/../stubs/') as $file) { + $this->publishes([ + $file->getRealPath() => base_path("stubs/filament/{$file->getFilename()}"), + ], 'filament-stubs'); + } + } + Testable::mixin(new TestsActions()); } + + /** + * @return array + */ + protected function getCommands(): array + { + $commands = [ + Commands\MakeImporterCommand::class, + ]; + + $aliases = []; + + foreach ($commands as $command) { + $class = 'Filament\\Actions\\Commands\\Aliases\\' . class_basename($command); + + if (! class_exists($class)) { + continue; + } + + $aliases[] = $class; + } + + return [ + ...$commands, + ...$aliases, + ]; + } } diff --git a/packages/actions/src/Commands/Aliases/MakeImporterCommand.php b/packages/actions/src/Commands/Aliases/MakeImporterCommand.php new file mode 100644 index 0000000000..adc5fa53a8 --- /dev/null +++ b/packages/actions/src/Commands/Aliases/MakeImporterCommand.php @@ -0,0 +1,12 @@ +getModel($model); + + if (blank($model)) { + return '//'; + } + + $table = $this->getModelTable($model); + + if (blank($table)) { + return '//'; + } + + $columns = []; + + foreach ($table->getColumns() as $column) { + if ($column->getAutoincrement()) { + continue; + } + + $columnName = $column->getName(); + + if (str($columnName)->is([ + app($model)->getKeyName(), + 'created_at', + 'deleted_at', + 'updated_at', + '*_token', + ])) { + continue; + } + + $columnData = []; + + if (in_array($columnName, [ + 'sku', + 'uuid', + ])) { + $columnData['label'] = [Str::upper($columnName)]; + } + + if ($column->getNotnull()) { + $columnData['rules'][0][] = 'required'; + $columnData['requiredMapping'] = []; + } + + if (str($columnName)->contains(['email'])) { + $columnData['rules'][0][] = 'email'; + } + + if ( + str($columnName)->endsWith('_id') && + filled($guessedRelationshipName = $this->guessBelongsToRelationshipName($column, $model)) + ) { + $columnName = $guessedRelationshipName; + $columnData['relationship'] = []; + } elseif (in_array($column->getType()::class, [ + Types\BooleanType::class, + ])) { + $columnData['rules'][0][] = 'boolean'; + $columnData['boolean'] = []; + } elseif (in_array($column->getType()::class, [ + Types\DateImmutableType::class, + Types\DateType::class, + ])) { + $columnData['rules'][0][] = 'date'; + } elseif (in_array($column->getType()::class, [ + Types\DateTimeImmutableType::class, + Types\DateTimeType::class, + Types\DateTimeTzImmutableType::class, + Types\DateTimeTzType::class, + ])) { + $columnData['rules'][0][] = 'datetime'; + } elseif (in_array($column->getType()::class, [ + Types\IntegerType::class, + Types\SmallIntType::class, + Types\BigIntType::class, + ])) { + $columnData['rules'][0][] = 'integer'; + $columnData['numeric'] = []; + } elseif (in_array($column->getType()::class, [ + Types\DecimalType::class, + Types\FloatType::class, + ])) { + $columnData['rules'][0][] = 'integer'; + $columnData['numeric'] = []; + } elseif ($length = $column->getLength()) { + $columnData['rules'][0][] = "max:{$length}"; + } + + // Move rules to the end of the column definition. + if (array_key_exists('rules', $columnData)) { + $rules = $columnData['rules']; + unset($columnData['rules']); + + $columnData['rules'] = $rules; + } + + $columns[$columnName] = $columnData; + } + + $output = count($columns) ? '' : '//'; + + foreach ($columns as $columnName => $columnData) { + // Constructor + $output .= 'ImportColumn::make(\''; + $output .= $columnName; + $output .= '\')'; + + // Configuration + foreach ($columnData as $methodName => $parameters) { + $output .= PHP_EOL; + $output .= ' ->'; + $output .= $methodName; + $output .= '('; + $output .= collect($parameters) + ->map(function (mixed $parameterValue, int | string $parameterName): string { + $parameterValue = match (true) { + /** @phpstan-ignore-next-line */ + is_bool($parameterValue) => $parameterValue ? 'true' : 'false', + /** @phpstan-ignore-next-line */ + is_null($parameterValue) => 'null', + is_numeric($parameterValue) => $parameterValue, + is_array($parameterValue) => '[\'' . implode('\', \'', $parameterValue) . '\']', + default => "'{$parameterValue}'", + }; + + if (is_numeric($parameterName)) { + return $parameterValue; + } + + return "{$parameterName}: {$parameterValue}"; + }) + ->implode(', '); + $output .= ')'; + } + + // Termination + $output .= ','; + + if (! (array_key_last($columns) === $columnName)) { + $output .= PHP_EOL; + } + } + + return $output; + } +} diff --git a/packages/actions/src/Commands/MakeImporterCommand.php b/packages/actions/src/Commands/MakeImporterCommand.php new file mode 100644 index 0000000000..5c28a445f0 --- /dev/null +++ b/packages/actions/src/Commands/MakeImporterCommand.php @@ -0,0 +1,86 @@ +argument('name') ?? text( + label: 'What is the model name?', + placeholder: 'BlogPost', + required: true, + )) + ->studly() + ->beforeLast('Importer') + ->trim('/') + ->trim('\\') + ->trim(' ') + ->studly() + ->replace('/', '\\'); + + if (blank($model)) { + $model = 'Importer'; + } + + $modelClass = (string) str($model)->afterLast('\\'); + $modelNamespace = str($model)->contains('\\') ? + (string) str($model)->beforeLast('\\') : + ''; + + $namespace = 'App\\Filament\\Imports'; + $path = app_path('Filament/Imports/'); + + $importer = "{$model}Importer"; + $importerClass = "{$modelClass}Importer"; + $importerNamespace = $modelNamespace; + $namespace .= $importerNamespace !== '' ? "\\{$importerNamespace}" : ''; + + $baseImporterPath = + (string) str($importer) + ->prepend('/') + ->prepend($path) + ->replace('\\', '/') + ->replace('//', '/'); + + $importerPath = "{$baseImporterPath}.php"; + + if (! $this->option('force') && $this->checkForCollision([ + $importerPath, + ])) { + return static::INVALID; + } + + $this->copyStubToApp('Importer', $importerPath, [ + 'columns' => $this->indentString($this->option('generate') ? $this->getImporterColumns( + 'App\\Models' . ($modelNamespace !== '' ? "\\{$modelNamespace}" : '') . '\\' . $modelClass, + ) : '//', 3), + 'model' => $model === 'Importer' ? 'Importer as ImporterModel' : $model, + 'modelClass' => $model === 'Importer' ? 'ImporterModel' : $modelClass, + 'modelLabel' => get_model_label($model), + 'namespace' => $namespace, + 'importerClass' => $importerClass, + ]); + + $this->components->info("Filament importer [{$importerPath}] created successfully."); + + return static::SUCCESS; + } +} diff --git a/packages/actions/src/ImportAction.php b/packages/actions/src/ImportAction.php new file mode 100644 index 0000000000..e0414b1e18 --- /dev/null +++ b/packages/actions/src/ImportAction.php @@ -0,0 +1,373 @@ + + */ + protected string $importer; + + protected ?string $job = null; + + protected int | Closure $chunkSize = 100; + + protected int | Closure | null $maxRows = null; + + protected function setUp(): void + { + parent::setUp(); + + $this->label(fn (ImportAction $action): string => __('filament-actions::import.label', ['label' => $action->getPluralModelLabel()])); + + $this->modalHeading(fn (ImportAction $action): string => __('filament-actions::import.modal.heading', ['label' => $action->getPluralModelLabel()])); + + $this->modalDescription(fn (ImportAction $action): Htmlable => new HtmlString('

' . $action->getModalAction('downloadExample')->toHtml() . '

')); + + $this->modalSubmitActionLabel(__('filament-actions::import.modal.actions.import.label')); + + $this->groupedIcon('heroicon-m-arrow-up-tray'); + + $this->form(fn (ImportAction $action): array => array_merge([ + FileUpload::make('file') + ->label(__('filament-actions::import.modal.form.file.label')) + ->placeholder(__('filament-actions::import.modal.form.file.placeholder')) + ->acceptedFileTypes(['text/csv', 'text/plain']) + ->afterStateUpdated(function (Forms\Set $set, ?TemporaryUploadedFile $state) use ($action) { + if (! $state instanceof TemporaryUploadedFile) { + return; + } + + $csvStream = $this->getUploadedFileStream($state); + + if (! $csvStream) { + return; + } + + $csvReader = CsvReader::createFromStream($csvStream); + $csvReader->setHeaderOffset(0); + + $csvColumns = $csvReader->getHeader(); + + $lowercaseCsvColumnValues = array_map('strtolower', $csvColumns); + $lowercaseCsvColumnKeys = array_combine( + $lowercaseCsvColumnValues, + $csvColumns, + ); + + $set('columnMap', array_reduce($action->getImporter()::getColumns(), function (array $carry, ImportColumn $column) use ($lowercaseCsvColumnKeys, $lowercaseCsvColumnValues) { + $carry[$column->getName()] = $lowercaseCsvColumnKeys[ + Arr::first( + array_intersect( + $lowercaseCsvColumnValues, + $column->getGuesses(), + ), + ) + ] ?? null; + + return $carry; + }, [])); + }) + ->storeFiles(false) + ->visibility('private') + ->required() + ->hiddenLabel(), + Fieldset::make(__('filament-actions::import.modal.form.columns.label')) + ->schema(function (Forms\Get $get) use ($action): array { + $csvFile = Arr::first((array) ($get('file') ?? [])); + + if (! $csvFile instanceof TemporaryUploadedFile) { + return []; + } + + $csvStream = $this->getUploadedFileStream($csvFile); + + if (! $csvStream) { + return []; + } + + $csvReader = CsvReader::createFromStream($csvStream); + $csvReader->setHeaderOffset(0); + + $csvColumns = $csvReader->getHeader(); + $csvColumnOptions = array_combine($csvColumns, $csvColumns); + + return array_map( + fn (ImportColumn $column): Select => $column->getSelect()->options($csvColumnOptions), + $action->getImporter()::getColumns(), + ); + }) + ->statePath('columnMap') + ->visible(fn (Forms\Get $get): bool => Arr::first((array) ($get('file') ?? [])) instanceof TemporaryUploadedFile), + ], $action->getImporter()::getOptionsFormComponents())); + + $this->action(function (ImportAction $action, array $data) { + /** @var TemporaryUploadedFile $csvFile */ + $csvFile = $data['file']; + + $csvStream = $this->getUploadedFileStream($csvFile); + + if (! $csvStream) { + return; + } + + $csvReader = CsvReader::createFromStream($csvStream); + $csvReader->setHeaderOffset(0); + $csvResults = Statement::create()->process($csvReader); + + $totalRows = $csvResults->count(); + $maxRows = $action->getMaxRows() ?? $totalRows; + + if ($maxRows < $totalRows) { + Notification::make() + ->title(__('filament-actions::import.notifications.max_rows.title')) + ->body(trans_choice('filament-actions::import.notifications.max_rows.body', $maxRows, [ + 'count' => format_number($maxRows), + ])) + ->success() + ->send(); + + return; + } + + $user = auth()->user(); + + $import = app(Import::class); + $import->user()->associate($user); + $import->file_name = $csvFile->getClientOriginalName(); + $import->file_path = $csvFile->getRealPath(); + $import->importer = $action->getImporter(); + $import->total_rows = $totalRows; + $import->save(); + + $importChunkIterator = new ChunkIterator($csvResults->getRecords(), chunkSize: $action->getChunkSize()); + + /** @var array>> $importChunks */ + $importChunks = $importChunkIterator->get(); + + $job = $action->getJob(); + + $importJobs = collect($importChunks) + ->map(fn (array $importChunk): object => new ($job)( + $import, + rows: $importChunk, + columnMap: $data['columnMap'], + options: Arr::except($data, ['file', 'columnMap']), + )); + + Bus::batch($importJobs->all()) + ->allowFailures() + ->finally(function () use ($import) { + if ($import->failed_at) { + return; + } + + $import->touch('completed_at'); + + if (! $import->user instanceof Authenticatable) { + return; + } + + $failedRowsCount = $import->getFailedRowsCount(); + + Notification::make() + ->title(__('filament-actions::import.notifications.completed.title')) + ->body($import->importer::getCompletedNotificationBody($import)) + ->when( + ! $failedRowsCount, + fn (Notification $notification) => $notification->success(), + ) + ->when( + $failedRowsCount && ($failedRowsCount < $import->total_rows), + fn (Notification $notification) => $notification->warning(), + ) + ->when( + $failedRowsCount === $import->total_rows, + fn (Notification $notification) => $notification->danger(), + ) + ->when( + $failedRowsCount, + fn (Notification $notification) => $notification->actions([ + NotificationAction::make('downloadFailedRowsCsv') + ->label(trans_choice('filament-actions::import.notifications.completed.download_failed_rows_csv.label', $failedRowsCount, [ + 'count' => format_number($failedRowsCount), + ])) + ->color('danger') + ->url(route('filament.imports.failed-rows.download', ['import' => $import])), + ]), + ) + ->sendToDatabase($import->user); + }) + ->dispatch(); + + Notification::make() + ->title($action->getSuccessNotificationTitle()) + ->body(trans_choice('filament-actions::import.notifications.started.body', $import->total_rows, [ + 'count' => format_number($import->total_rows), + ])) + ->success() + ->send(); + }); + + $this->registerModalActions([ + Action::make('downloadExample') + ->label(__('filament-actions::import.modal.actions.download_example.label')) + ->link() + ->action(function (): StreamedResponse { + $columns = $this->getImporter()::getColumns(); + + $csv = Writer::createFromFileObject(new SplTempFileObject()); + + $csv->insertOne(array_map( + fn (ImportColumn $column): string => $column->getName(), + $columns, + )); + + $example = array_map( + fn (ImportColumn $column) => $column->getExample(), + $columns, + ); + + if (array_filter( + $example, + fn ($value): bool => filled($value), + )) { + $csv->insertOne($example); + } + + return response()->streamDownload(function () use ($csv) { + echo $csv->toString(); + }, __('filament-actions::import.example_csv.file_name', ['importer' => (string) str($this->getImporter())->classBasename()->kebab()]), [ + 'Content-Type' => 'text/csv', + ]); + }), + ]); + + $this->color('gray'); + + $this->modalWidth('xl'); + + $this->successNotificationTitle(__('filament-actions::import.notifications.started.title')); + + $this->model(fn (ImportAction $action): string => $action->getImporter()::getModel()); + } + + /** + * @return resource | false + */ + public function getUploadedFileStream(TemporaryUploadedFile $file) + { + $filePath = $file->getRealPath(); + + if (config('filament.default_filesystem_disk') !== 's3') { + return fopen($filePath, mode: 'r'); + } + + /** @var AwsS3V3Adapter $s3Adapter */ + $s3Adapter = Storage::disk('s3')->getAdapter(); + + invade($s3Adapter)->client->registerStreamWrapper(); + + $fileS3Path = 's3://' . config('filesystems.disks.s3.bucket') . '/' . $filePath; + + return fopen($fileS3Path, mode: 'r', context: stream_context_create([ + 's3' => [ + 'seekable' => true, + ], + ])); + } + + public static function getDefaultName(): ?string + { + return 'import'; + } + + /** + * @param class-string $importer + */ + public function importer(string $importer): static + { + $this->importer = $importer; + + return $this; + } + + /** + * @param class-string | null $job + */ + public function job(?string $job): static + { + $this->job = $job; + + return $this; + } + + public function chunkSize(int | Closure $size): static + { + $this->size = $size; + + return $this; + } + + public function max(int | Closure | null $rows): static + { + $this->maxRows = $rows; + + return $this; + } + + /** + * @return class-string + */ + public function getImporter(): string + { + return $this->importer; + } + + /** + * @return class-string + */ + public function getJob(): string + { + return $this->job ?? ImportCsv::class; + } + + public function getChunkSize(): int + { + return $this->evaluate($this->chunkSize); + } + + public function getMaxRows(): ?int + { + return $this->evaluate($this->maxRows); + } +} diff --git a/packages/actions/src/Imports/Http/Controllers/DownloadImportFailureCsv.php b/packages/actions/src/Imports/Http/Controllers/DownloadImportFailureCsv.php new file mode 100644 index 0000000000..df5ba8a911 --- /dev/null +++ b/packages/actions/src/Imports/Http/Controllers/DownloadImportFailureCsv.php @@ -0,0 +1,46 @@ +user->is(auth()->user()), 403); + + $csv = Writer::createFromFileObject(new SplTempFileObject()); + + $columnHeaders = array_keys($import->failedRows()->first()->data); + $columnHeaders[] = __('filament-actions::import.failure_csv.error_header'); + + $csv->insertOne($columnHeaders); + + $import->failedRows() + ->lazyById(100) + ->each(fn (FailedImportRow $failedImportRow) => $csv->insertOne([ + ...$failedImportRow->data, + 'error' => $failedImportRow->validation_error ?? __('filament-actions::import.failure_csv.system_error'), + ])); + + return response()->streamDownload(function () use ($csv) { + foreach ($csv->chunk(1000) as $offset => $chunk) { + echo $chunk; + + if ($offset % 1000) { + flush(); + } + } + }, __('filament-actions::import.failure_csv.file_name', [ + 'import_id' => $import->getKey(), + 'csv_name' => $import->file_name, + ]) . '.csv', [ + 'Content-Type' => 'text/csv', + ]); + } +} diff --git a/packages/actions/src/Imports/ImportColumn.php b/packages/actions/src/Imports/ImportColumn.php new file mode 100644 index 0000000000..bbb56b5208 --- /dev/null +++ b/packages/actions/src/Imports/ImportColumn.php @@ -0,0 +1,498 @@ + | Closure + */ + protected array | Closure $guesses = []; + + protected ?Closure $fillRecordUsing = null; + + protected ?Closure $sanitizeStateUsing = null; + + /** + * @var array | Closure + */ + protected array | Closure $dataValidationRules = []; + + /** + * @var array | Closure + */ + protected array | Closure $nestedRecursiveDataValidationRules = []; + + protected ?Importer $importer = null; + + protected mixed $example = null; + + protected string | Closure | null $relationship = null; + + /** + * @var string | array | Closure | null + */ + protected string | array | Closure | null $resolveRelationshipUsing = null; + + /** + * @var array + */ + protected array $resolvedRelatedRecords = []; + + final public function __construct(string $name) + { + $this->name($name); + } + + public static function make(string $name): static + { + $static = app(static::class, ['name' => $name]); + $static->configure(); + + return $static; + } + + public function getSelect(): Select + { + return Select::make($this->getName()) + ->label($this->label) + ->placeholder(__('filament-actions::import.modal.form.columns.placeholder')) + ->required($this->isMappingRequired); + } + + public function name(string $name): static + { + $this->name = $name; + + return $this; + } + + public function label(string | Closure | null $label): static + { + $this->label = $label; + + return $this; + } + + public function example(mixed $example): static + { + $this->example = $example; + + return $this; + } + + public function requiredMapping(bool | Closure $condition = true): static + { + $this->isMappingRequired = $condition; + + return $this; + } + + public function numeric(bool | Closure $condition = true, int | Closure | null $decimalPlaces = null): static + { + $this->isNumeric = $condition; + $this->decimalPlaces = $decimalPlaces; + + return $this; + } + + public function boolean(bool | Closure $condition = true): static + { + $this->isBoolean = $condition; + + return $this; + } + + public function ignoreBlankState(bool | Closure $condition = true): static + { + $this->isBlankStateIgnored = $condition; + + return $this; + } + + /** + * @param array | Closure $rules + */ + public function rules(array | Closure $rules): static + { + $this->dataValidationRules = $rules; + + return $this; + } + + /** + * @param array | Closure $rules + */ + public function nestedRecursiveRules(array | Closure $rules): static + { + $this->nestedRecursiveDataValidationRules = $rules; + + return $this; + } + + public function array(string | Closure | null $separator = ','): static + { + $this->arraySeparator = $separator; + + return $this; + } + + /** + * @param array | Closure $guesses + */ + public function guess(array | Closure $guesses): static + { + $this->guesses = $guesses; + + return $this; + } + + public function importer(?Importer $importer): static + { + $this->importer = $importer; + + return $this; + } + + /** + * @return array + */ + public function getGuesses(): array + { + $guesses = $this->evaluate($this->guesses); + array_unshift($guesses, $this->getName()); + + return array_reduce($guesses, function (array $carry, string $guess): array { + $guess = (string) Str::of($guess) + ->lower() + ->replace('-', ' ') + ->replace('_', ' '); + $carry[] = $guess; + + if (Str::of($guess)->contains(' ')) { + $carry[] = (string) Str::of($guess)->replace(' ', '-'); + $carry[] = (string) Str::of($guess)->replace(' ', '_'); + } + + return $carry; + }, []); + } + + public function sanitizeStateUsing(?Closure $callback): static + { + $this->sanitizeStateUsing = $callback; + + return $this; + } + + public function fillRecordUsing(?Closure $callback): static + { + $this->fillRecordUsing = $callback; + + return $this; + } + + /** + * @param array $options + */ + public function sanitizeState(mixed $state, array $options): mixed + { + $originalState = $state; + + if (filled($arraySeparator = $this->getArraySeparator())) { + $state = collect(explode($arraySeparator, strval($state))) + ->map(fn (mixed $stateItem): mixed => $this->sanitizeStateItem($stateItem)) + ->filter(fn (mixed $stateItem): bool => filled($stateItem)) + ->all(); + } else { + $state = $this->sanitizeStateItem($state); + } + + if ($this->sanitizeStateUsing) { + return $this->evaluate($this->sanitizeStateUsing, [ + 'originalState' => $originalState, + 'state' => $state, + 'options' => $options, + ]); + } + + return $state; + } + + public function fillRecord(mixed $state): void + { + if ($this->fillRecordUsing) { + $this->evaluate($this->fillRecordUsing, [ + 'state' => $state, + ]); + + return; + } + + $relationship = $this->getRelationship(); + + if ($relationship) { + $relationship->associate($this->resolveRelatedRecord($state)); + + return; + } + + $this->getRecord()->{$this->getName()} = $state; + } + + public function getName(): string + { + return $this->name; + } + + /** + * @return array + */ + public function getDataValidationRules(): array + { + $rules = $this->evaluate($this->dataValidationRules); + + if ($this->hasRelationship()) { + $rules[] = function (string $attribute, mixed $state, Closure $fail) { + if (blank($state)) { + return; + } + + $record = $this->resolveRelatedRecord($state); + + if ($record) { + return; + } + + $fail(__('validation.exists', ['attribute' => $attribute])); + }; + } + + return $rules; + } + + public function resolveRelatedRecord(mixed $state): ?Model + { + if (array_key_exists($state, $this->resolvedRelatedRecords)) { + return $this->resolvedRelatedRecords[$state]; + } + + /** @var BelongsTo $relationship */ + $relationship = Relation::noConstraints(fn () => $this->getRelationship()); + $relationshipQuery = $relationship->getQuery(); + + if (blank($this->resolveRelationshipUsing)) { + return $this->resolvedRelatedRecords[$state] = $relationshipQuery + ->where($relationship->getQualifiedOwnerKeyName(), $state) + ->first(); + } + + $resolveUsing = $this->evaluate($this->resolveRelationshipUsing, [ + 'state' => $state, + ]); + + if ($resolveUsing instanceof Model) { + return $this->resolvedRelatedRecords[$state] = $resolveUsing; + } + + if (! (is_array($resolveUsing) || is_string($resolveUsing))) { + return null; + } + + $resolveUsing = Arr::wrap($resolveUsing); + + $isFirst = true; + + foreach ($resolveUsing as $columnToResolve) { + $whereClause = $isFirst ? 'where' : 'orWhere'; + + $relationshipQuery->{$whereClause}( + $columnToResolve, + $state, + ); + + $isFirst = false; + } + + return $this->resolvedRelatedRecords[$state] = $relationshipQuery->first(); + } + + /** + * @return array + */ + public function getNestedRecursiveDataValidationRules(): array + { + return $this->evaluate($this->nestedRecursiveDataValidationRules); + } + + public function isNumeric(): bool + { + return (bool) $this->evaluate($this->isNumeric); + } + + public function isBoolean(): bool + { + return (bool) $this->evaluate($this->isBoolean); + } + + public function isBlankStateIgnored(): bool + { + return (bool) $this->evaluate($this->isBlankStateIgnored); + } + + public function getDecimalPlaces(): ?int + { + return $this->evaluate($this->decimalPlaces); + } + + public function getArraySeparator(): ?string + { + return $this->evaluate($this->arraySeparator); + } + + public function isArray(): bool + { + return filled($this->getArraySeparator()); + } + + public function getImporter(): ?Importer + { + return $this->importer; + } + + public function getExample(): mixed + { + return $this->evaluate($this->example); + } + + /** + * @param string | array | Closure | null $resolveUsing + */ + public function relationship(string | Closure | null $name = null, string | array | Closure | null $resolveUsing = null): static + { + $this->relationship = $name ?? $this->getName(); + $this->resolveRelationshipUsing = $resolveUsing; + + return $this; + } + + public function getRelationship(): ?BelongsTo + { + $name = $this->getRelationshipName(); + + if (blank($name)) { + return null; + } + + return $this->getRecord()->{$name}(); + } + + public function getRelationshipName(): ?string + { + return $this->evaluate($this->relationship); + } + + public function getRecord(): ?Model + { + return $this->getImporter()->getRecord(); + } + + public function hasRelationship(): bool + { + return filled($this->getRelationshipName()); + } + + protected function sanitizeStateItem(mixed $state): mixed + { + if (is_string($state)) { + $state = trim($state); + } + + if (blank($state)) { + return null; + } + + if ($this->isBoolean()) { + return $this->sanitizeBooleanStateItem($state); + } + + if ($this->isNumeric()) { + return $this->sanitizeNumericStateItem($state); + } + + return $state; + } + + protected function sanitizeBooleanStateItem(mixed $state): bool + { + // Narrow down the possible values of the state to make comparison easier. + $state = strtolower(strval($state)); + + return match ($state) { + '1', 'true', 'yes', 'y', 'on' => true, + '0', 'false', 'no', 'n', 'off' => false, + default => (bool) $state, + }; + } + + protected function sanitizeNumericStateItem(mixed $state): int | float + { + $state = floatval(preg_replace('/[^0-9.]/', '', $state)); + + $decimalPlaces = $this->getDecimalPlaces(); + + if ($decimalPlaces === null) { + return $state; + } + + return round($state, $decimalPlaces); + } + + protected function resolveDefaultClosureDependencyForEvaluationByName(string $parameterName): array + { + return match ($parameterName) { + 'data' => [$this->getImporter()->getData()], + 'importer' => [$this->getImporter()], + 'options' => [$this->getImporter()->getOptions()], + 'originalData' => [$this->getImporter()->getOriginalData()], + 'record' => [$this->getRecord()], + default => parent::resolveDefaultClosureDependencyForEvaluationByName($parameterName), + }; + } + + protected function resolveDefaultClosureDependencyForEvaluationByType(string $parameterType): array + { + $record = $this->getRecord(); + + return match ($parameterType) { + Importer::class => [$this->getImporter()], + Model::class, $record ? $record::class : null => [$record], + default => parent::resolveDefaultClosureDependencyForEvaluationByType($parameterType), + }; + } +} diff --git a/packages/actions/src/Imports/Importer.php b/packages/actions/src/Imports/Importer.php new file mode 100644 index 0000000000..6476ee0c9b --- /dev/null +++ b/packages/actions/src/Imports/Importer.php @@ -0,0 +1,295 @@ + */ + protected array $cachedColumns; + + /** + * @var array + */ + protected array $originalData; + + /** + * @var array + */ + protected array $data; + + protected ?Model $record; + + protected static ?string $model = null; + + /** + * @param array $columnMap + * @param array $options + */ + public function __construct( + readonly protected Import $import, + readonly protected array $columnMap, + readonly protected array $options, + ) { + } + + /** + * @param array $data + */ + public function __invoke(array $data): void + { + $this->originalData = $this->data = $data; + $this->record = null; + + $this->remapData(); + $this->sanitizeData(); + + $this->record = $this->resolveRecord(); + + if (! $this->record) { + return; + } + + $this->callHook('beforeValidate'); + $this->validateData(); + $this->callHook('afterValidate'); + + $this->callHook('beforeFill'); + $this->fillRecord(); + $this->callHook('afterFill'); + + $recordExists = $this->record->exists; + + $this->callHook('beforeSave'); + $this->callHook($recordExists ? 'beforeUpdate' : 'beforeCreate'); + $this->saveRecord(); + $this->callHook('afterSave'); + $this->callHook($recordExists ? 'afterUpdate' : 'afterCreate'); + + $this->import->increment('successful_rows'); + } + + public function remapData(): void + { + $data = $this->data; + + foreach ($this->getCachedColumns() as $column) { + $columnName = $column->getName(); + $rowColumnName = $this->columnMap[$columnName] ?? null; + + if (! array_key_exists($rowColumnName, $this->data)) { + continue; + } + + $data[$columnName] = $this->data[$rowColumnName]; + } + + $this->data = $data; + } + + public function sanitizeData(): void + { + foreach ($this->getCachedColumns() as $column) { + $columnName = $column->getName(); + + if (! array_key_exists($columnName, $this->data)) { + continue; + } + + $this->data[$columnName] = $column->sanitizeState( + $this->data[$columnName], + $this->options, + ); + } + } + + public function resolveRecord(): ?Model + { + $keyName = app(static::getModel())->getKeyName(); + $keyColumnName = $this->columnMap[$keyName] ?? $keyName; + + return static::getModel()::find($this->data[$keyColumnName]); + } + + /** + * @throws ValidationException + */ + public function validateData(): void + { + $validator = Validator::make( + $this->data, + $this->getValidationRules(), + $this->getValidationMessages(), + $this->getValidationAttributes(), + ); + + $validator->validate(); + } + + /** + * @return array> + */ + public function getValidationRules(): array + { + $rules = []; + + foreach ($this->getCachedColumns() as $column) { + $columnName = $column->getName(); + + $rules[$columnName] = $column->getDataValidationRules(); + + if ( + $column->isArray() && + count($nestedRecursiveRules = $column->getNestedRecursiveDataValidationRules()) + ) { + $rules["{$columnName}.*"] = $nestedRecursiveRules; + } + } + + return $rules; + } + + /** + * @return array + */ + public function getValidationMessages(): array + { + return []; + } + + /** + * @return array + */ + public function getValidationAttributes(): array + { + return []; + } + + public function fillRecord(): void + { + foreach ($this->getCachedColumns() as $column) { + $columnName = $column->getName(); + + if (! array_key_exists($columnName, $this->data)) { + continue; + } + + $state = $this->data[$columnName]; + + if (blank($state) && $column->isBlankStateIgnored()) { + continue; + } + + $column->fillRecord($state); + } + } + + public function saveRecord(): void + { + $this->record->save(); + } + + /** + * @return array + */ + abstract public static function getColumns(): array; + + /** + * @return array + */ + public static function getOptionsFormComponents(): array + { + return []; + } + + /** + * @return class-string + */ + public static function getModel(): string + { + return static::$model ?? (string) str(class_basename(static::class)) + ->beforeLast('Importer') + ->prepend('App\\Models\\'); + } + + abstract public static function getCompletedNotificationBody(Import $import): string; + + /** + * @return array + */ + public function getJobMiddleware(): array + { + return [ + (new WithoutOverlapping("import{$this->import->id}"))->expireAfter(600), + ]; + } + + public function getJobRetryUntil(): CarbonInterface + { + return now()->addDay(); + } + + /** + * @return array + */ + public function getJobTags(): array + { + return ["import{$this->import->id}"]; + } + + /** + * @return array + */ + public function getCachedColumns(): array + { + return $this->cachedColumns ??= array_map( + fn (ImportColumn $column) => $column->importer($this), + static::getColumns(), + ); + } + + public function getRecord(): ?Model + { + return $this->record; + } + + /** + * @return array + */ + public function getOriginalData(): array + { + return $this->originalData; + } + + /** + * @return array + */ + public function getData(): array + { + return $this->data; + } + + /** + * @return array + */ + public function getOptions(): array + { + return $this->options; + } + + protected function callHook(string $hook): void + { + if (! method_exists($this, $hook)) { + return; + } + + $this->{$hook}(); + } +} diff --git a/packages/actions/src/Imports/Jobs/ImportCsv.php b/packages/actions/src/Imports/Jobs/ImportCsv.php new file mode 100644 index 0000000000..ff700a1eb6 --- /dev/null +++ b/packages/actions/src/Imports/Jobs/ImportCsv.php @@ -0,0 +1,125 @@ +> $rows + * @param array $columnMap + * @param array $options + */ + public function __construct( + readonly public Import $import, + readonly public array $rows, + readonly public array $columnMap, + readonly public array $options = [], + ) { + $this->importer = $this->import->getImporter( + $this->columnMap, + $this->options, + ); + } + + /** + * @return array + */ + public function middleware(): array + { + return $this->importer->getJobMiddleware(); + } + + public function handle(): void + { + /** @var Authenticatable $user */ + $user = $this->import->user; + + auth()->login($user); + + $exceptions = []; + + foreach ($this->rows as $row) { + try { + DB::transaction(fn () => ($this->importer)($row)); + } catch (ValidationException $exception) { + $this->logFailedRow($row, collect($exception->errors())->flatten()->implode(' ')); + } catch (Throwable $exception) { + $exceptions[$exception::class] = $exception; + + $this->logFailedRow($row); + } + + $this->import->increment('processed_rows'); + } + + $this->handleExceptions($exceptions); + } + + public function retryUntil(): CarbonInterface + { + return $this->importer->getJobRetryUntil(); + } + + /** + * @return array + */ + public function tags(): array + { + return $this->importer->getJobTags(); + } + + /** + * @param array $data + */ + protected function logFailedRow(array $data, ?string $validationError = null): void + { + $failedRow = app(FailedImportRow::class); + $failedRow->import()->associate($this->import); + $failedRow->data = $data; + $failedRow->validation_error = $validationError; + $failedRow->save(); + } + + /** + * @param array $exceptions + */ + protected function handleExceptions(array $exceptions): void + { + if (empty($exceptions)) { + return; + } + + if (count($exceptions) > 1) { + throw new Exception('Multiple types of exceptions occurred: [' . implode('], [', array_keys($exceptions)) . ']'); + } + + throw Arr::first($exceptions); + } +} diff --git a/packages/actions/src/Imports/Models/FailedImportRow.php b/packages/actions/src/Imports/Models/FailedImportRow.php new file mode 100644 index 0000000000..adda2241c6 --- /dev/null +++ b/packages/actions/src/Imports/Models/FailedImportRow.php @@ -0,0 +1,38 @@ + 'array', + ]; + + protected $guarded = []; + + public function import(): BelongsTo + { + return $this->belongsTo(app(Import::class)::class); + } + + public function prunable(): Builder + { + return static::where( + 'created_at', + '<=', + now()->subMonth(), + ); + } +} diff --git a/packages/actions/src/Imports/Models/Import.php b/packages/actions/src/Imports/Models/Import.php new file mode 100644 index 0000000000..98b927184e --- /dev/null +++ b/packages/actions/src/Imports/Models/Import.php @@ -0,0 +1,65 @@ + $importer + * @property int $processed_rows + * @property int $total_rows + * @property int $successful_rows + * @property-read Collection $failedRows + * @property-read Authenticatable $user + */ +class Import extends Model +{ + protected $casts = [ + 'completed_at' => 'timestamp', + 'processed_rows' => 'integer', + 'total_rows' => 'integer', + 'successful_rows' => 'integer', + ]; + + protected $guarded = []; + + public function failedRows(): HasMany + { + return $this->hasMany(app(FailedImportRow::class)::class); + } + + public function user(): BelongsTo + { + return $this->belongsTo(app(Authenticatable::class)::class); + } + + /** + * @param array $columnMap + * @param array $options + */ + public function getImporter( + array $columnMap, + array $options, + ): Importer { + return app($this->importer, [ + 'import' => $this, + 'columnMap' => $columnMap, + 'options' => $options, + ]); + } + + public function getFailedRowsCount(): int + { + return $this->total_rows - $this->successful_rows; + } +} diff --git a/packages/actions/stubs/Importer.stub b/packages/actions/stubs/Importer.stub new file mode 100644 index 0000000000..bf02802bbd --- /dev/null +++ b/packages/actions/stubs/Importer.stub @@ -0,0 +1,41 @@ +data['column_name']` + // 'email' => $this->data['email'], + // ]); + + return new {{ modelClass }}(); + } + + public static function getCompletedNotificationBody(Import $import): string + { + $body = 'Your {{ modelLabel }} import has completed and ' . number_format($import->successful_rows) . ' ' . str('row')->plural($import->successful_rows) . ' imported.'; + + if ($failedRowsCount = $import->getFailedRowsCount()) { + $body .= ' ' . number_format($failedRowsCount) . ' ' . str('row')->plural($failedRowsCount) . ' failed to import.'; + } + + return $body; + } +} diff --git a/packages/support/src/ChunkIterator.php b/packages/support/src/ChunkIterator.php new file mode 100644 index 0000000000..5255ca25a9 --- /dev/null +++ b/packages/support/src/ChunkIterator.php @@ -0,0 +1,38 @@ +iterator->valid(); $i++) { + $chunk[] = $this->iterator->current(); + + $this->iterator->next(); + + if (count($chunk) !== $this->chunkSize) { + continue; + } + + yield $chunk; + + $chunk = []; + } + + if (count($chunk)) { + yield $chunk; + } + } +}