AMartinez_CustomImportExport - Version 1.5.011

Version Notes

Magento 1.5 version.

Download this release

Release Info

Developer Antonio Martinez
Extension AMartinez_CustomImportExport
Version 1.5.011
Comparing to
See all releases


Code changes from version 1.5.010 to 1.5.011

app/code/community/AMartinez/CustomImportExport/Model/Import/Adapter/Csv1.php CHANGED
@@ -62,8 +62,7 @@ class AMartinez_CustomImportExport_Model_Import_Adapter_Csv1 extends Mage_Import
62
  {
63
  $this->_fileHandler = fopen($this->_source, 'r');
64
  $this->rewind();
65
-
66
- echo ":::: " . Mage::helper('importexport')->__("Multivalued column names: ") . implode(',', $this->_multiValuedColNames) . " ::::\n";
67
 
68
  return $this;
69
  }
62
  {
63
  $this->_fileHandler = fopen($this->_source, 'r');
64
  $this->rewind();
65
+ //echo ":::: " . Mage::helper('importexport')->__("Multivalued column names: ") . implode(',', $this->_multiValuedColNames) . " ::::\n";
 
66
 
67
  return $this;
68
  }
app/code/community/AMartinez/CustomImportExport/etc/config.xml CHANGED
@@ -14,7 +14,7 @@
14
 
15
  <modules>
16
  <AMartinez_CustomImportExport>
17
- <version>1.5.008</version>
18
  <depends>
19
  <!-- no dependencies -->
20
  </depends>
14
 
15
  <modules>
16
  <AMartinez_CustomImportExport>
17
+ <version>1.5.011</version>
18
  <depends>
19
  <!-- no dependencies -->
20
  </depends>
app/code/community/AMartinez/CustomImportExport/shell/run.php CHANGED
@@ -53,38 +53,10 @@ class AMartinez_CustomImportExport extends Mage_Shell_Abstract
53
 
54
  // products import
55
  if ($filename = $this->getSourceFile()) {
56
- // line count
57
- $linecount = ($this->getArg('l') ? $this->getArg('l') : ($this->getArg('linecount') ? $this->getArg('linecount') : true));
58
- $extension = pathinfo($filename, PATHINFO_EXTENSION);
59
- $count = 0;
60
- $i = 0;
61
- $j = 0;
62
- $files = array();
63
- $h = fopen($filename, 'r');
64
- $header = fgets($h);
65
- while($line = fgets($h))
66
- {
67
- // first iteration / no "attr-only" line / line count has been reached
68
- if ($i == 0 || (substr($line, 0 ,1) != ',' && substr($line, 0 ,2) != '""' && substr($line, 0 ,2) != "''" && $i > $linecount && !($linecount === true)))
69
- {
70
- if (defined('t'))
71
- {
72
- fclose($t);
73
- }
74
- $files[] = $filename . '.tmp.' . $j . '.' . $extension;
75
- $t = fopen($files[$j++], 'w');
76
- fwrite($t, $header);
77
- $i = 0;
78
- }
79
- fwrite($t, $line);
80
- $count++;
81
- $i++;
82
- }
83
- fclose($h);
84
- fclose($t);
85
-
86
- echo "Start '$this->behavior' action (products source file: $filename, raw line count: $count)\n";
87
- Mage::log("Start '$this->behavior' action (products source file: $filename, raw line count: $count)", Zend_Log::DEBUG);
88
 
89
  $count = 0;
90
  foreach ($files as $file)
@@ -108,6 +80,7 @@ class AMartinez_CustomImportExport extends Mage_Shell_Abstract
108
  }
109
 
110
  if ($message) {
 
111
  Mage::throwException(sprintf("File %s contains %s corrupt records (from a total of %s)", $file, $import->getInvalidRowsCount(), $processedRowsCount) . $message);
112
  }
113
 
@@ -117,10 +90,10 @@ class AMartinez_CustomImportExport extends Mage_Shell_Abstract
117
  }
118
 
119
  $count += $processedRowsCount;
120
- $import->importSource();
121
  }
122
- unlink($file);
123
  }
 
124
  echo "\nDone (processed rows count: " . $count . ")\n";
125
  }
126
 
@@ -144,31 +117,74 @@ class AMartinez_CustomImportExport extends Mage_Shell_Abstract
144
  }
145
 
146
  // customers import
147
- if ($file = $this->getCustomersSourceFile()) {
148
- echo "Start '$this->behavior' action. Customers source file $file\n";
149
- Mage::log("Start '$this->behavior' action. Customers source file $file", Zend_Log::DEBUG);
150
-
151
- $customersimport = $this->initCustomersImportModel();
152
- $validationResult = $customersimport->validateSource($file);
153
- $processedRowsCount = $customersimport->getProcessedRowsCount();
154
-
155
- if ($processedRowsCount > 0)
156
- {
157
- if (!$validationResult)
 
 
 
 
 
158
  {
159
- $message = sprintf("File %s contains %s corrupt records (from a total of %s)",
160
- $file, $customersimport->getInvalidRowsCount(), $processedRowsCount
161
- );
162
- foreach ($customersimport->getErrors() as $type => $lines)
163
  {
164
- $message .= "\n:::: " . $type . " ::::\nIn Line(s) " . implode(", ", $lines) . "\n";
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
165
  }
166
- Mage::throwException($message);
 
 
167
  }
168
-
169
- $customersimport->importSource();
170
- }
171
- echo "Done (processed rows count: " . $processedRowsCount . ")\n";
 
172
  }
173
 
174
  // customers export
@@ -495,6 +511,56 @@ class AMartinez_CustomImportExport extends Mage_Shell_Abstract
495
  return false;
496
  }
497
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
498
 
499
  /**
500
  * Retrieve usage help message
@@ -502,7 +568,7 @@ class AMartinez_CustomImportExport extends Mage_Shell_Abstract
502
  public function usageHelp()
503
  {
504
  return <<<HELP
505
- AMartinez_CustomImportExport script (v. 1.5.010)
506
 
507
  NAME
508
  run.php
@@ -533,7 +599,7 @@ OPTIONS
533
 
534
  -a
535
  -all
536
- same as -c -f -p -r
537
 
538
  -b
539
  -behavior
@@ -580,12 +646,21 @@ OPTIONS
580
  reindex data by all indexers (attributes, prices, etc)
581
 
582
  EXAMPLES
583
- php -f app/code/community/AMartinez/CustomImportExport/shell/run.php -- -import var/importexport/products.csv
584
- php -f app/code/community/AMartinez/CustomImportExport/shell/run.php -- -all
585
- php -f app/code/community/AMartinez/CustomImportExport/shell/run.php -- -a -i var/importexport/products.csv
586
- php -f app/code/community/AMartinez/CustomImportExport/shell/run.php -- -e var/importexport/products.csv
587
- php -f app/code/community/AMartinez/CustomImportExport/shell/run.php -- -databasebackup -import -all
588
-
 
 
 
 
 
 
 
 
 
589
  CREDITS
590
  I would be delighted to hear from you if you like this script. I also appreciate any donation you wish to contribute to the growth of this project (paypal: toniyecla [at] gmail [dot] com).
591
 
53
 
54
  // products import
55
  if ($filename = $this->getSourceFile()) {
56
+ $files = $this->splitFile($filename, ($this->getArg('l') ? $this->getArg('l') : ($this->getArg('linecount') ? $this->getArg('linecount') : true)));
57
+
58
+ echo "Start products '$this->behavior' action from $filename\n";
59
+ Mage::log("Start proucts '$this->behavior' action from $filename", Zend_Log::DEBUG);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
60
 
61
  $count = 0;
62
  foreach ($files as $file)
80
  }
81
 
82
  if ($message) {
83
+ $this->unlinkFiles($files);
84
  Mage::throwException(sprintf("File %s contains %s corrupt records (from a total of %s)", $file, $import->getInvalidRowsCount(), $processedRowsCount) . $message);
85
  }
86
 
90
  }
91
 
92
  $count += $processedRowsCount;
93
+ $import->importSource();
94
  }
 
95
  }
96
+ $this->unlinkFiles($files);
97
  echo "\nDone (processed rows count: " . $count . ")\n";
98
  }
99
 
117
  }
118
 
119
  // customers import
120
+ if ($filename = $this->getCustomersSourceFile()) {
121
+ // get only one field each time to prevent disruption because global errors
122
+ $files = $this->splitFile($filename, ($this->getArg('l') ? $this->getArg('l') : ($this->getArg('linecount') ? $this->getArg('linecount') : true)));
123
+
124
+ echo "Start customers '$this->behavior' action from $filename\n";
125
+ Mage::log("Start customers '$this->behavior' action from $filename", Zend_Log::DEBUG);
126
+
127
+ $count = 0;
128
+ $files_ = array();
129
+ foreach ($files as $file)
130
+ {
131
+ $customersimport = $this->initCustomersImportModel();
132
+ $validationResult = $customersimport->validateSource($file);
133
+ $processedRowsCount = $customersimport->getProcessedRowsCount();
134
+
135
+ if ($processedRowsCount > 0)
136
  {
137
+ while (!$validationResult)
 
 
 
138
  {
139
+ $errorLines = array();
140
+ foreach ($customersimport->getErrors() as $type => $lines)
141
+ {
142
+ echo ":::: " . $type . " in line(s) " . implode(", ", $lines) . "::::\n";
143
+ $errorLines = array_merge($errorLines, $lines);
144
+ }
145
+ $file_ = str_replace('tmp.', 'tmp._', $file);
146
+ $files_[] = $file_;
147
+ $h = fopen($file, 'r');
148
+ $t = fopen($file_, 'w');
149
+ $i = 0;
150
+ $error = false;
151
+ while($line = fgets($h))
152
+ {
153
+ if ((substr($line, 0 ,1) != ',' && substr($line, 0 ,2) != '""' && substr($line, 0 ,2) != "''"))
154
+ {
155
+ $error = false;
156
+ }
157
+ if (!$error)
158
+ {
159
+ if (!in_array($i, $errorLines))
160
+ {
161
+ fwrite($t, $line);
162
+ }
163
+ else
164
+ {
165
+ echo ":::: Line " . $i . " :::: " . $line;
166
+ $error = true;
167
+ }
168
+ }
169
+ $i++;
170
+ }
171
+ fclose($h);
172
+ fclose($t);
173
+
174
+ $customersimport = $this->initCustomersImportModel();
175
+ $validationResult = $customersimport->validateSource($file_);
176
+ $processedRowsCount = $customersimport->getProcessedRowsCount();
177
+ $file = $file_;
178
  }
179
+
180
+ $count += $processedRowsCount;
181
+ $customersimport->importSource();
182
  }
183
+ }
184
+
185
+ $this->unlinkFiles($files);
186
+ $this->unlinkFiles($files_);
187
+ echo "\nDone (processed rows count: " . $count . ")\n";
188
  }
189
 
190
  // customers export
511
  return false;
512
  }
513
  }
514
+
515
+ /**
516
+ * Split file into pieces
517
+ */
518
+ public function splitFile($filename, $linecount)
519
+ {
520
+ $extension = pathinfo($filename, PATHINFO_EXTENSION);
521
+ $count = 0;
522
+ $i = 0;
523
+ $j = 0;
524
+ $files = array();
525
+ $h = fopen($filename, 'r');
526
+ $header = fgets($h);
527
+ while($line = fgets($h))
528
+ {
529
+ // first iteration / no "attr-only" line / line count has been reached
530
+ if ($i == 0 || (substr($line, 0 ,1) != ',' && substr($line, 0 ,2) != '""' && substr($line, 0 ,2) != "''" && $i >= $linecount && !($linecount === true)))
531
+ {
532
+ if (defined('t'))
533
+ {
534
+ fclose($t);
535
+ }
536
+ $files[] = $filename . '.tmp.' . $j . '.' . $extension;
537
+ $t = fopen($files[$j++], 'w');
538
+ fwrite($t, $header);
539
+ $i = 0;
540
+ }
541
+ fwrite($t, $line);
542
+ $count++;
543
+ $i++;
544
+ }
545
+ fclose($h);
546
+ if (defined('t'))
547
+ {
548
+ fclose($t);
549
+ }
550
+
551
+ return $files;
552
+ }
553
+
554
+ /**
555
+ * Unlink temp files
556
+ */
557
+ public function unlinkFiles($files)
558
+ {
559
+ foreach ($files as $file)
560
+ {
561
+ unlink($file);
562
+ }
563
+ }
564
 
565
  /**
566
  * Retrieve usage help message
568
  public function usageHelp()
569
  {
570
  return <<<HELP
571
+ AMartinez_CustomImportExport script (v. 1.5.011)
572
 
573
  NAME
574
  run.php
599
 
600
  -a
601
  -all
602
+ same as -c -f -p -r options
603
 
604
  -b
605
  -behavior
646
  reindex data by all indexers (attributes, prices, etc)
647
 
648
  EXAMPLES
649
+ ussual import
650
+ php -f app/code/community/AMartinez/CustomImportExport/shell/run.php -- -all -import var/customimportexport/test_single_row_style.csv1
651
+
652
+ ussual export
653
+ php -f app/code/community/AMartinez/CustomImportExport/shell/run.php -- -e
654
+
655
+ reindex / flush cache
656
+ php -f app/code/community/AMartinez/CustomImportExport/shell/run.php -- -a
657
+
658
+ backup, then import
659
+ php -f app/code/community/AMartinez/CustomImportExport/shell/run.php -- -databasebackup -import -all
660
+
661
+ crontab example (daily)
662
+ 00 03 * * * root php -f /var/www/html/magento/store/app/code/community/AMartinez/CustomImportExport/shell/run.php -- -db -a -i /var/www/html/magento/store/var/customimportexport/products.csv1 | xargs -0 -I msg echo -e "Subject: MY STORE products daily update \n\n " msg | /usr/sbin/sendmail -f some@address.com some@address.com
663
+
664
  CREDITS
665
  I would be delighted to hear from you if you like this script. I also appreciate any donation you wish to contribute to the growth of this project (paypal: toniyecla [at] gmail [dot] com).
666
 
package.xml CHANGED
@@ -1,7 +1,7 @@
1
  <?xml version="1.0"?>
2
  <package>
3
  <name>AMartinez_CustomImportExport</name>
4
- <version>1.5.010</version>
5
  <stability>stable</stability>
6
  <license uri="http://opensource.org/licenses/osl-3.0">OSL v3.0</license>
7
  <channel>community</channel>
@@ -95,9 +95,9 @@ CREDITS&#xD;
95
  </description>
96
  <notes>Magento 1.5 version.</notes>
97
  <authors><author><name>Antonio Martinez</name><user>amartinez</user><email>toniyecla@gmail.com</email></author></authors>
98
- <date>2011-05-18</date>
99
- <time>11:16:33</time>
100
- <contents><target name="magecommunity"><dir name="AMartinez"><dir name="CustomImportExport"><dir name="Helper"><file name="Data.php" hash="798c6c9b7adc8e5a66c8d00d7d475485"/></dir><dir name="Model"><file name="Backup.php" hash="0e673d6ab8a39f33bd1185b6627fb742"/><file name="Export.php" hash="252ef36d1b5e93e9f38e8d45dd72b030"/><dir name="Import"><dir name="Adapter"><file name="Csv1.php" hash="3615d594c083da04af8596b2658ff7a6"/></dir><file name="Adapter.php" hash="b2ddc64699cab599061789ff066faf11"/><dir name="Entity"><dir name="Customer"><file name="Address.php" hash="e06e286e132d4d2ec8548ebc66f257e1"/></dir><file name="Customer.php" hash="555a87242f1a613629c5d14bde5d23b8"/><dir name="Product"><dir name="Type"><file name="Configurable.php" hash="88c4c1db1c38ad8ae5a310ee19c87eee"/><file name="Grouped.php" hash="cfebd1d4520c25776ea02d2f3b980a82"/><file name="Simple.php" hash="a58046af6ab03b3a10f0ae575abf3a27"/></dir></dir><file name="Product.php" hash="4fa62cf9a2206d161f8135cf1908e305"/></dir></dir><file name="Import.php" hash="6d182eb456ea8785d0571f640efbdaac"/></dir><dir name="etc"><file name="config.xml" hash="6a502554a8eeb6e878bfacfc4a339e74"/></dir><dir name="shell"><file name="run.php" hash="3204b4cb9d3f8cbba4652f5791a07e75"/></dir></dir></dir></target><target name="mageetc"><dir name="modules"><file name="AMartinez_CustomImportExport.xml" hash="75ba4fbc265d1cf2b6ab61b8f3a3b95e"/></dir></target><target name="mage"><dir name="var"><dir name="customimportexport"><file name="test_configurable.csv" hash="d0447d047ef94f2f65a9fd375348cd39"/><file name="test_grouped.csv" hash="9c76b71b05c8e2b52efae4725f4119a4"/><file name="test_multiple_stores.csv1" hash="72c5977fcac5ce7ae433cf2f4f09c6d2"/><file name="test_single_row_style.csv1" hash="dfcd266cb65ab97e93f9402e4f32bffb"/></dir></dir></target></contents>
101
  <compatible/>
102
  <dependencies><required><php><min>5.2.0</min><max>6.0.0</max></php></required></dependencies>
103
  </package>
1
  <?xml version="1.0"?>
2
  <package>
3
  <name>AMartinez_CustomImportExport</name>
4
+ <version>1.5.011</version>
5
  <stability>stable</stability>
6
  <license uri="http://opensource.org/licenses/osl-3.0">OSL v3.0</license>
7
  <channel>community</channel>
95
  </description>
96
  <notes>Magento 1.5 version.</notes>
97
  <authors><author><name>Antonio Martinez</name><user>amartinez</user><email>toniyecla@gmail.com</email></author></authors>
98
+ <date>2011-05-24</date>
99
+ <time>09:10:30</time>
100
+ <contents><target name="magecommunity"><dir name="AMartinez"><dir name="CustomImportExport"><dir name="Helper"><file name="Data.php" hash="798c6c9b7adc8e5a66c8d00d7d475485"/></dir><dir name="Model"><file name="Backup.php" hash="0e673d6ab8a39f33bd1185b6627fb742"/><file name="Export.php" hash="252ef36d1b5e93e9f38e8d45dd72b030"/><dir name="Import"><dir name="Adapter"><file name="Csv1.php" hash="0f3cd525811041fedeaf1234719d9485"/></dir><file name="Adapter.php" hash="b2ddc64699cab599061789ff066faf11"/><dir name="Entity"><dir name="Customer"><file name="Address.php" hash="e06e286e132d4d2ec8548ebc66f257e1"/></dir><file name="Customer.php" hash="555a87242f1a613629c5d14bde5d23b8"/><dir name="Product"><dir name="Type"><file name="Configurable.php" hash="88c4c1db1c38ad8ae5a310ee19c87eee"/><file name="Grouped.php" hash="cfebd1d4520c25776ea02d2f3b980a82"/><file name="Simple.php" hash="a58046af6ab03b3a10f0ae575abf3a27"/></dir></dir><file name="Product.php" hash="4fa62cf9a2206d161f8135cf1908e305"/></dir></dir><file name="Import.php" hash="6d182eb456ea8785d0571f640efbdaac"/></dir><dir name="etc"><file name="config.xml" hash="da233e3a255d676f364f41605a67d837"/></dir><dir name="shell"><file name="run.php" hash="4501bd199a6097ca3a29b442cb8f03d0"/></dir></dir></dir></target><target name="mageetc"><dir name="modules"><file name="AMartinez_CustomImportExport.xml" hash="75ba4fbc265d1cf2b6ab61b8f3a3b95e"/></dir></target><target name="mage"><dir name="var"><dir name="customimportexport"><file name="test_configurable.csv" hash="d0447d047ef94f2f65a9fd375348cd39"/><file name="test_grouped.csv" hash="9c76b71b05c8e2b52efae4725f4119a4"/><file name="test_multiple_stores.csv1" hash="72c5977fcac5ce7ae433cf2f4f09c6d2"/><file name="test_single_row_style.csv1" hash="ac873b64867a36641fdd84297897dff5"/></dir></dir></target></contents>
101
  <compatible/>
102
  <dependencies><required><php><min>5.2.0</min><max>6.0.0</max></php></required></dependencies>
103
  </package>
var/customimportexport/test_single_row_style.csv1 CHANGED
@@ -2,4 +2,4 @@
2
  "TEST1-S1-35",,"simple","base","Default","2011-4-8","Shoes","TEST COLLECTION","1","1","1.10","1","999","35","43.90","1899-12-31","2","TEST1-S1 (35)","TEST COLLECTION","TEST COLLECTION",,,,
3
  "TEST1-S1-36",,"simple","base","Default","2011-4-8","Shoes","TEST COLLECTION","1","1","1.10","1","999","36","43.90","1899-12-31","2","TEST1-S1 (36)","TEST COLLECTION","TEST COLLECTION",,,,
4
  "TEST1-S1-37",,"simple","base","Default","2011-4-8","Shoes","TEST COLLECTION","1","1","1.10","1","999","37","43.90","1899-12-31","2","TEST1-S1 (37)","TEST COLLECTION","TEST COLLECTION",,,,
5
- "TEST1-S1",,"grouped","base","Default","2011-4-8","Shoes","TEST COLLECTION,MALICIA","1","4","1.10","1","13986","","43.90","1899-12-31","2","TEST 1 ANTIK NEGRO, PISO TEST test","TEST COLLECTION","TEST COLLECTION","TEST1-S1-35,TEST1-S1-36,TEST1-S1-37","/img_articulos/TEST1-S1.jpg","/img_articulos/TEST1-S1.jpg","/img_articulos/TEST1-S1.jpg"
2
  "TEST1-S1-35",,"simple","base","Default","2011-4-8","Shoes","TEST COLLECTION","1","1","1.10","1","999","35","43.90","1899-12-31","2","TEST1-S1 (35)","TEST COLLECTION","TEST COLLECTION",,,,
3
  "TEST1-S1-36",,"simple","base","Default","2011-4-8","Shoes","TEST COLLECTION","1","1","1.10","1","999","36","43.90","1899-12-31","2","TEST1-S1 (36)","TEST COLLECTION","TEST COLLECTION",,,,
4
  "TEST1-S1-37",,"simple","base","Default","2011-4-8","Shoes","TEST COLLECTION","1","1","1.10","1","999","37","43.90","1899-12-31","2","TEST1-S1 (37)","TEST COLLECTION","TEST COLLECTION",,,,
5
+ "TEST1-S1",,"grouped","base","Default","2011-4-8","Shoes","TEST COLLECTION/SUBTEST,MALICIA","1","4","1.10","1","13986","","43.90","1899-12-31","2","TEST 1 ANTIK NEGRO, PISO TEST test","TEST COLLECTION","TEST COLLECTION","TEST1-S1-35,TEST1-S1-36,TEST1-S1-37","/img_articulos/TEST1-S1.jpg","/img_articulos/TEST1-S1.jpg","/img_articulos/TEST1-S1.jpg"