Matomo Analytics – Ethical Stats. Powerful Insights.

Version Description

Release Info

Developer	matomoteam
Plugin	Matomo Analytics – Ethical Stats. Powerful Insights.
Version	1.0.2
Comparing to
See all releases

Version 1.0.2

Files changed (129) hide show

.htaccess +49 -0
LEGALNOTICE +45 -0
LICENSE +675 -0
app/.htaccess +5 -0
app/LEGALNOTICE +295 -0
app/LICENSE +675 -0
app/PRIVACY.md +60 -0
app/README.md +119 -0
app/SECURITY.md +23 -0
app/bootstrap.php +202 -0
app/config/environment/dev.php +12 -0
app/config/global.ini.php +1033 -0
app/config/global.php +221 -0
app/console +32 -0
app/core/.htaccess +24 -0
app/core/API/ApiRenderer.php +152 -0
app/core/API/CORSHandler.php +57 -0
app/core/API/DataTableGenericFilter.php +245 -0
app/core/API/DataTableManipulator.php +208 -0
app/core/API/DataTableManipulator/Flattener.php +236 -0
app/core/API/DataTableManipulator/LabelFilter.php +222 -0
app/core/API/DataTableManipulator/ReportTotalsCalculator.php +251 -0
app/core/API/DataTablePostProcessor.php +502 -0
app/core/API/DocumentationGenerator.php +397 -0
app/core/API/Inconsistencies.php +47 -0
app/core/API/Proxy.php +582 -0
app/core/API/Request.php +653 -0
app/core/API/ResponseBuilder.php +245 -0
app/core/Access.php +739 -0
app/core/Access/CapabilitiesProvider.php +123 -0
app/core/Access/Capability.php +29 -0
app/core/Access/Role.php +22 -0
app/core/Access/Role/Admin.php +40 -0
app/core/Access/Role/View.php +39 -0
app/core/Access/Role/Write.php +38 -0
app/core/Access/RolesProvider.php +62 -0
app/core/Application/Environment.php +252 -0
app/core/Application/EnvironmentManipulator.php +59 -0
app/core/Application/Kernel/EnvironmentValidator.php +153 -0
app/core/Application/Kernel/GlobalSettingsProvider.php +111 -0
app/core/Application/Kernel/PluginList.php +197 -0
app/core/Archive.php +906 -0
app/core/Archive/ArchiveInvalidator.php +481 -0
app/core/Archive/ArchiveInvalidator/InvalidationResult.php +56 -0
app/core/Archive/ArchivePurger.php +338 -0
app/core/Archive/ArchiveQuery.php +49 -0
app/core/Archive/ArchiveQueryFactory.php +127 -0
app/core/Archive/Chunk.php +144 -0
app/core/Archive/DataCollection.php +377 -0
app/core/Archive/DataTableFactory.php +594 -0
app/core/Archive/Parameters.php +59 -0
app/core/ArchiveProcessor.php +640 -0
app/core/ArchiveProcessor/ArchivingStatus.php +111 -0
app/core/ArchiveProcessor/Loader.php +245 -0
app/core/ArchiveProcessor/Parameters.php +261 -0
app/core/ArchiveProcessor/PluginsArchiver.php +339 -0
app/core/ArchiveProcessor/PluginsArchiverException.php +15 -0
app/core/ArchiveProcessor/Rules.php +321 -0
app/core/Archiver/Request.php +101 -0
app/core/AssetManager.php +448 -0
app/core/AssetManager/UIAsset.php +61 -0
app/core/AssetManager/UIAsset/InMemoryUIAsset.php +62 -0
app/core/AssetManager/UIAsset/OnDiskUIAsset.php +135 -0
app/core/AssetManager/UIAssetCacheBuster.php +66 -0
app/core/AssetManager/UIAssetCatalog.php +73 -0
app/core/AssetManager/UIAssetCatalogSorter.php +58 -0
app/core/AssetManager/UIAssetFetcher.php +150 -0
app/core/AssetManager/UIAssetFetcher/JScriptUIAssetFetcher.php +88 -0
app/core/AssetManager/UIAssetFetcher/StaticUIAssetFetcher.php +36 -0
app/core/AssetManager/UIAssetFetcher/StylesheetUIAssetFetcher.php +85 -0
app/core/AssetManager/UIAssetMerger.php +193 -0
app/core/AssetManager/UIAssetMerger/JScriptUIAssetMerger.php +88 -0
app/core/AssetManager/UIAssetMerger/StylesheetUIAssetMerger.php +260 -0
app/core/AssetManager/UIAssetMinifier.php +66 -0
app/core/Auth.php +221 -0
app/core/Auth/Password.php +69 -0
app/core/BaseFactory.php +65 -0
app/core/Cache.php +117 -0
app/core/CacheId.php +87 -0
app/core/Category/Category.php +124 -0
app/core/Category/CategoryList.php +95 -0
app/core/Category/Subcategory.php +146 -0
app/core/CliMulti.php +471 -0
app/core/CliMulti/CliPhp.php +107 -0
app/core/CliMulti/Output.php +76 -0
app/core/CliMulti/Process.php +273 -0
app/core/CliMulti/RequestCommand.php +129 -0
app/core/Columns/ComputedMetricFactory.php +57 -0
app/core/Columns/Dimension.php +907 -0
app/core/Columns/DimensionMetricFactory.php +122 -0
app/core/Columns/DimensionsProvider.php +62 -0
app/core/Columns/Discriminator.php +75 -0
app/core/Columns/Join.php +61 -0
app/core/Columns/Join/ActionNameJoin.php +24 -0
app/core/Columns/Join/GoalNameJoin.php +24 -0
app/core/Columns/Join/SiteNameJoin.php +24 -0
app/core/Columns/MetricsList.php +191 -0
app/core/Columns/Updater.php +380 -0
app/core/Common.php +1331 -0
app/core/Composer/ScriptHandler.php +47 -0
app/core/Concurrency/DistributedList.php +171 -0
app/core/Concurrency/Lock.php +138 -0
app/core/Concurrency/LockBackend.php +58 -0
app/core/Concurrency/LockBackend/MySqlLockBackend.php +146 -0
app/core/Config.php +478 -0
app/core/Config/Cache.php +89 -0
app/core/Config/ConfigNotFoundException.php +16 -0
app/core/Config/IniFileChain.php +539 -0
app/core/Console.php +319 -0
app/core/Container/ContainerDoesNotExistException.php +18 -0
app/core/Container/ContainerFactory.php +152 -0
app/core/Container/IniConfigDefinitionSource.php +95 -0
app/core/Container/StaticContainer.php +87 -0
app/core/Context.php +94 -0
app/core/Cookie.php +462 -0
app/core/CronArchive.php +2192 -0
app/core/CronArchive/FixedSiteIds.php +65 -0
app/core/CronArchive/Performance/Logger.php +119 -0
app/core/CronArchive/Performance/Measurement.php +165 -0
app/core/CronArchive/SegmentArchivingRequestUrlProvider.php +208 -0
app/core/CronArchive/SharedSiteIds.php +202 -0
app/core/CronArchive/SitesToReprocessDistributedList.php +40 -0
app/core/DataAccess/Actions.php +34 -0
app/core/DataAccess/ArchiveSelector.php +384 -0
app/core/DataAccess/ArchiveTableCreator.php +123 -0
app/core/DataAccess/ArchiveTableDao.php +90 -0
app/core/DataAccess/ArchiveWriter.php +303 -0
app/core/DataAccess/ArchivingDbAdapter.php +107 -0
app/core/DataAccess/LogAggregator.php +1016 -0

.htaccess ADDED Viewed

	@@ -0,0 +1,49 @@


1	+ # This file is provided from Matomo Analytics, do not edit directly
2	+ # Please report any issue or improvement directly to the Matomo team.
3	+ # Do not allow access to any php file directly unless it is index/matomo.php
4	+ <Files ~ "(\.php)$">
5	+ <IfModule mod_version.c>
6	+ <IfVersion < 2.4>
7	+ Order allow,deny
8	+ Deny from all
9	+ </IfVersion>
10	+ <IfVersion >= 2.4>
11	+ Require all denied
12	+ </IfVersion>
13	+ </IfModule>
14	+ <IfModule !mod_version.c>
15	+ <IfModule !mod_authz_core.c>
16	+ Order allow,deny
17	+ Deny from all
18	+ </IfModule>
19	+ <IfModule mod_authz_core.c>
20	+ Require all denied
21	+ </IfModule>
22	+ </IfModule>
23	+ </Files>
24	+ <Files ~ "^((index\|piwik\|matomo)\.php)$">
25	+ <IfModule mod_version.c>
26	+ <IfVersion < 2.4>
27	+ Order allow,deny
28	+ Allow from all
29	+ </IfVersion>
30	+ <IfVersion >= 2.4>
31	+ Require all granted
32	+ </IfVersion>
33	+ </IfModule>
34	+ <IfModule !mod_version.c>
35	+ <IfModule !mod_authz_core.c>
36	+ Order allow,deny
37	+ Allow from all
38	+ </IfModule>
39	+ <IfModule mod_authz_core.c>
40	+ Require all granted
41	+ </IfModule>
42	+ </IfModule>
43	+ </Files>
44	+
45	+ # Serve HTML files as text/html mime type - Note: requires mod_mime apache module!
46	+ <IfModule mod_mime.c>
47	+ AddHandler text/html .html
48	+ AddHandler text/html .htm
49	+ </IfModule>

LEGALNOTICE ADDED Viewed

	@@ -0,0 +1,45 @@


1	+ SOFTWARE LICENSE
2	+
3	+ The free software license of Matomo is GNU General Public License v3
4	+ or later. A copy of GNU GPL v3 should have been included in this
5	+ software package in LICENSE.
6	+
7	+
8	+ TRADEMARK
9	+
10	+ Matomo (TM) is an internationally registered trademark.
11	+
12	+ The software license does not grant any rights under trademark
13	+ law for use of the trademark. Refer to https://matomo.org/trademark/
14	+ for up-to-date trademark licensing information.
15	+
16	+ *
17	+ * The software license applies to both the aggregate software and
18	+ * application-specific portions of the software.
19	+ *
20	+ * You may not remove this legal notice or modify the software
21	+ * in such a way that misrepresents the origin of the software.
22	+ *
23	+
24	+ CREDITS
25	+
26	+ The software consists of contributions made by many individuals.
27	+ Major contributors are listed in https://matomo.org/team/.
28	+
29	+ For detailed contribution history, refer to the source, tickets,
30	+ patches, and Git revision history, available at
31	+ https://github.com/matomo-org/wp-matomo/issues
32	+ https://github.com/matomo-org/wp-matomo
33	+
34	+
35	+ SEPARATELY LICENSED COMPONENTS AND LIBRARIES
36	+
37	+ See a list of all components/libraries and its licenses in Matomo in `app/LEGALNOTICE`
38	+
39	+ THIRD-PARTY COMPONENTS AND LIBRARIES
40	+
41	+ See a list of all components/libraries and its licenses in Matomo in `app/LEGALNOTICE`
42	+
43	+ THIRD-PARTY CONTENT
44	+
45	+ See the list in `app/LEGALNOTICE`

LICENSE ADDED Viewed

	@@ -0,0 +1,675 @@


1	+ GNU GENERAL PUBLIC LICENSE
2	+ Version 3, 29 June 2007
3	+
4	+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
5	+ Everyone is permitted to copy and distribute verbatim copies
6	+ of this license document, but changing it is not allowed.
7	+
8	+ Preamble
9	+
10	+ The GNU General Public License is a free, copyleft license for
11	+ software and other kinds of works.
12	+
13	+ The licenses for most software and other practical works are designed
14	+ to take away your freedom to share and change the works. By contrast,
15	+ the GNU General Public License is intended to guarantee your freedom to
16	+ share and change all versions of a program--to make sure it remains free
17	+ software for all its users. We, the Free Software Foundation, use the
18	+ GNU General Public License for most of our software; it applies also to
19	+ any other work released this way by its authors. You can apply it to
20	+ your programs, too.
21	+
22	+ When we speak of free software, we are referring to freedom, not
23	+ price. Our General Public Licenses are designed to make sure that you
24	+ have the freedom to distribute copies of free software (and charge for
25	+ them if you wish), that you receive source code or can get it if you
26	+ want it, that you can change the software or use pieces of it in new
27	+ free programs, and that you know you can do these things.
28	+
29	+ To protect your rights, we need to prevent others from denying you
30	+ these rights or asking you to surrender the rights. Therefore, you have
31	+ certain responsibilities if you distribute copies of the software, or if
32	+ you modify it: responsibilities to respect the freedom of others.
33	+
34	+ For example, if you distribute copies of such a program, whether
35	+ gratis or for a fee, you must pass on to the recipients the same
36	+ freedoms that you received. You must make sure that they, too, receive
37	+ or can get the source code. And you must show them these terms so they
38	+ know their rights.
39	+
40	+ Developers that use the GNU GPL protect your rights with two steps:
41	+ (1) assert copyright on the software, and (2) offer you this License
42	+ giving you legal permission to copy, distribute and/or modify it.
43	+
44	+ For the developers' and authors' protection, the GPL clearly explains
45	+ that there is no warranty for this free software. For both users' and
46	+ authors' sake, the GPL requires that modified versions be marked as
47	+ changed, so that their problems will not be attributed erroneously to
48	+ authors of previous versions.
49	+
50	+ Some devices are designed to deny users access to install or run
51	+ modified versions of the software inside them, although the manufacturer
52	+ can do so. This is fundamentally incompatible with the aim of
53	+ protecting users' freedom to change the software. The systematic
54	+ pattern of such abuse occurs in the area of products for individuals to
55	+ use, which is precisely where it is most unacceptable. Therefore, we
56	+ have designed this version of the GPL to prohibit the practice for those
57	+ products. If such problems arise substantially in other domains, we
58	+ stand ready to extend this provision to those domains in future versions
59	+ of the GPL, as needed to protect the freedom of users.
60	+
61	+ Finally, every program is threatened constantly by software patents.
62	+ States should not allow patents to restrict development and use of
63	+ software on general-purpose computers, but in those that do, we wish to
64	+ avoid the special danger that patents applied to a free program could
65	+ make it effectively proprietary. To prevent this, the GPL assures that
66	+ patents cannot be used to render the program non-free.
67	+
68	+ The precise terms and conditions for copying, distribution and
69	+ modification follow.
70	+
71	+ TERMS AND CONDITIONS
72	+
73	+ 0. Definitions.
74	+
75	+ "This License" refers to version 3 of the GNU General Public License.
76	+
77	+ "Copyright" also means copyright-like laws that apply to other kinds of
78	+ works, such as semiconductor masks.
79	+
80	+ "The Program" refers to any copyrightable work licensed under this
81	+ License. Each licensee is addressed as "you". "Licensees" and
82	+ "recipients" may be individuals or organizations.
83	+
84	+ To "modify" a work means to copy from or adapt all or part of the work
85	+ in a fashion requiring copyright permission, other than the making of an
86	+ exact copy. The resulting work is called a "modified version" of the
87	+ earlier work or a work "based on" the earlier work.
88	+
89	+ A "covered work" means either the unmodified Program or a work based
90	+ on the Program.
91	+
92	+ To "propagate" a work means to do anything with it that, without
93	+ permission, would make you directly or secondarily liable for
94	+ infringement under applicable copyright law, except executing it on a
95	+ computer or modifying a private copy. Propagation includes copying,
96	+ distribution (with or without modification), making available to the
97	+ public, and in some countries other activities as well.
98	+
99	+ To "convey" a work means any kind of propagation that enables other
100	+ parties to make or receive copies. Mere interaction with a user through
101	+ a computer network, with no transfer of a copy, is not conveying.
102	+
103	+ An interactive user interface displays "Appropriate Legal Notices"
104	+ to the extent that it includes a convenient and prominently visible
105	+ feature that (1) displays an appropriate copyright notice, and (2)
106	+ tells the user that there is no warranty for the work (except to the
107	+ extent that warranties are provided), that licensees may convey the
108	+ work under this License, and how to view a copy of this License. If
109	+ the interface presents a list of user commands or options, such as a
110	+ menu, a prominent item in the list meets this criterion.
111	+
112	+ 1. Source Code.
113	+
114	+ The "source code" for a work means the preferred form of the work
115	+ for making modifications to it. "Object code" means any non-source
116	+ form of a work.
117	+
118	+ A "Standard Interface" means an interface that either is an official
119	+ standard defined by a recognized standards body, or, in the case of
120	+ interfaces specified for a particular programming language, one that
121	+ is widely used among developers working in that language.
122	+
123	+ The "System Libraries" of an executable work include anything, other
124	+ than the work as a whole, that (a) is included in the normal form of
125	+ packaging a Major Component, but which is not part of that Major
126	+ Component, and (b) serves only to enable use of the work with that
127	+ Major Component, or to implement a Standard Interface for which an
128	+ implementation is available to the public in source code form. A
129	+ "Major Component", in this context, means a major essential component
130	+ (kernel, window system, and so on) of the specific operating system
131	+ (if any) on which the executable work runs, or a compiler used to
132	+ produce the work, or an object code interpreter used to run it.
133	+
134	+ The "Corresponding Source" for a work in object code form means all
135	+ the source code needed to generate, install, and (for an executable
136	+ work) run the object code and to modify the work, including scripts to
137	+ control those activities. However, it does not include the work's
138	+ System Libraries, or general-purpose tools or generally available free
139	+ programs which are used unmodified in performing those activities but
140	+ which are not part of the work. For example, Corresponding Source
141	+ includes interface definition files associated with source files for
142	+ the work, and the source code for shared libraries and dynamically
143	+ linked subprograms that the work is specifically designed to require,
144	+ such as by intimate data communication or control flow between those
145	+ subprograms and other parts of the work.
146	+
147	+ The Corresponding Source need not include anything that users
148	+ can regenerate automatically from other parts of the Corresponding
149	+ Source.
150	+
151	+ The Corresponding Source for a work in source code form is that
152	+ same work.
153	+
154	+ 2. Basic Permissions.
155	+
156	+ All rights granted under this License are granted for the term of
157	+ copyright on the Program, and are irrevocable provided the stated
158	+ conditions are met. This License explicitly affirms your unlimited
159	+ permission to run the unmodified Program. The output from running a
160	+ covered work is covered by this License only if the output, given its
161	+ content, constitutes a covered work. This License acknowledges your
162	+ rights of fair use or other equivalent, as provided by copyright law.
163	+
164	+ You may make, run and propagate covered works that you do not
165	+ convey, without conditions so long as your license otherwise remains
166	+ in force. You may convey covered works to others for the sole purpose
167	+ of having them make modifications exclusively for you, or provide you
168	+ with facilities for running those works, provided that you comply with
169	+ the terms of this License in conveying all material for which you do
170	+ not control copyright. Those thus making or running the covered works
171	+ for you must do so exclusively on your behalf, under your direction
172	+ and control, on terms that prohibit them from making any copies of
173	+ your copyrighted material outside their relationship with you.
174	+
175	+ Conveying under any other circumstances is permitted solely under
176	+ the conditions stated below. Sublicensing is not allowed; section 10
177	+ makes it unnecessary.
178	+
179	+ 3. Protecting Users' Legal Rights From Anti-Circumvention Law.
180	+
181	+ No covered work shall be deemed part of an effective technological
182	+ measure under any applicable law fulfilling obligations under article
183	+ 11 of the WIPO copyright treaty adopted on 20 December 1996, or
184	+ similar laws prohibiting or restricting circumvention of such
185	+ measures.
186	+
187	+ When you convey a covered work, you waive any legal power to forbid
188	+ circumvention of technological measures to the extent such circumvention
189	+ is effected by exercising rights under this License with respect to
190	+ the covered work, and you disclaim any intention to limit operation or
191	+ modification of the work as a means of enforcing, against the work's
192	+ users, your or third parties' legal rights to forbid circumvention of
193	+ technological measures.
194	+
195	+ 4. Conveying Verbatim Copies.
196	+
197	+ You may convey verbatim copies of the Program's source code as you
198	+ receive it, in any medium, provided that you conspicuously and
199	+ appropriately publish on each copy an appropriate copyright notice;
200	+ keep intact all notices stating that this License and any
201	+ non-permissive terms added in accord with section 7 apply to the code;
202	+ keep intact all notices of the absence of any warranty; and give all
203	+ recipients a copy of this License along with the Program.
204	+
205	+ You may charge any price or no price for each copy that you convey,
206	+ and you may offer support or warranty protection for a fee.
207	+
208	+ 5. Conveying Modified Source Versions.
209	+
210	+ You may convey a work based on the Program, or the modifications to
211	+ produce it from the Program, in the form of source code under the
212	+ terms of section 4, provided that you also meet all of these conditions:
213	+
214	+ a) The work must carry prominent notices stating that you modified
215	+ it, and giving a relevant date.
216	+
217	+ b) The work must carry prominent notices stating that it is
218	+ released under this License and any conditions added under section
219	+ 7. This requirement modifies the requirement in section 4 to
220	+ "keep intact all notices".
221	+
222	+ c) You must license the entire work, as a whole, under this
223	+ License to anyone who comes into possession of a copy. This
224	+ License will therefore apply, along with any applicable section 7
225	+ additional terms, to the whole of the work, and all its parts,
226	+ regardless of how they are packaged. This License gives no
227	+ permission to license the work in any other way, but it does not
228	+ invalidate such permission if you have separately received it.
229	+
230	+ d) If the work has interactive user interfaces, each must display
231	+ Appropriate Legal Notices; however, if the Program has interactive
232	+ interfaces that do not display Appropriate Legal Notices, your
233	+ work need not make them do so.
234	+
235	+ A compilation of a covered work with other separate and independent
236	+ works, which are not by their nature extensions of the covered work,
237	+ and which are not combined with it such as to form a larger program,
238	+ in or on a volume of a storage or distribution medium, is called an
239	+ "aggregate" if the compilation and its resulting copyright are not
240	+ used to limit the access or legal rights of the compilation's users
241	+ beyond what the individual works permit. Inclusion of a covered work
242	+ in an aggregate does not cause this License to apply to the other
243	+ parts of the aggregate.
244	+
245	+ 6. Conveying Non-Source Forms.
246	+
247	+ You may convey a covered work in object code form under the terms
248	+ of sections 4 and 5, provided that you also convey the
249	+ machine-readable Corresponding Source under the terms of this License,
250	+ in one of these ways:
251	+
252	+ a) Convey the object code in, or embodied in, a physical product
253	+ (including a physical distribution medium), accompanied by the
254	+ Corresponding Source fixed on a durable physical medium
255	+ customarily used for software interchange.
256	+
257	+ b) Convey the object code in, or embodied in, a physical product
258	+ (including a physical distribution medium), accompanied by a
259	+ written offer, valid for at least three years and valid for as
260	+ long as you offer spare parts or customer support for that product
261	+ model, to give anyone who possesses the object code either (1) a
262	+ copy of the Corresponding Source for all the software in the
263	+ product that is covered by this License, on a durable physical
264	+ medium customarily used for software interchange, for a price no
265	+ more than your reasonable cost of physically performing this
266	+ conveying of source, or (2) access to copy the
267	+ Corresponding Source from a network server at no charge.
268	+
269	+ c) Convey individual copies of the object code with a copy of the
270	+ written offer to provide the Corresponding Source. This
271	+ alternative is allowed only occasionally and noncommercially, and
272	+ only if you received the object code with such an offer, in accord
273	+ with subsection 6b.
274	+
275	+ d) Convey the object code by offering access from a designated
276	+ place (gratis or for a charge), and offer equivalent access to the
277	+ Corresponding Source in the same way through the same place at no
278	+ further charge. You need not require recipients to copy the
279	+ Corresponding Source along with the object code. If the place to
280	+ copy the object code is a network server, the Corresponding Source
281	+ may be on a different server (operated by you or a third party)
282	+ that supports equivalent copying facilities, provided you maintain
283	+ clear directions next to the object code saying where to find the
284	+ Corresponding Source. Regardless of what server hosts the
285	+ Corresponding Source, you remain obligated to ensure that it is
286	+ available for as long as needed to satisfy these requirements.
287	+
288	+ e) Convey the object code using peer-to-peer transmission, provided
289	+ you inform other peers where the object code and Corresponding
290	+ Source of the work are being offered to the general public at no
291	+ charge under subsection 6d.
292	+
293	+ A separable portion of the object code, whose source code is excluded
294	+ from the Corresponding Source as a System Library, need not be
295	+ included in conveying the object code work.
296	+
297	+ A "User Product" is either (1) a "consumer product", which means any
298	+ tangible personal property which is normally used for personal, family,
299	+ or household purposes, or (2) anything designed or sold for incorporation
300	+ into a dwelling. In determining whether a product is a consumer product,
301	+ doubtful cases shall be resolved in favor of coverage. For a particular
302	+ product received by a particular user, "normally used" refers to a
303	+ typical or common use of that class of product, regardless of the status
304	+ of the particular user or of the way in which the particular user
305	+ actually uses, or expects or is expected to use, the product. A product
306	+ is a consumer product regardless of whether the product has substantial
307	+ commercial, industrial or non-consumer uses, unless such uses represent
308	+ the only significant mode of use of the product.
309	+
310	+ "Installation Information" for a User Product means any methods,
311	+ procedures, authorization keys, or other information required to install
312	+ and execute modified versions of a covered work in that User Product from
313	+ a modified version of its Corresponding Source. The information must
314	+ suffice to ensure that the continued functioning of the modified object
315	+ code is in no case prevented or interfered with solely because
316	+ modification has been made.
317	+
318	+ If you convey an object code work under this section in, or with, or
319	+ specifically for use in, a User Product, and the conveying occurs as
320	+ part of a transaction in which the right of possession and use of the
321	+ User Product is transferred to the recipient in perpetuity or for a
322	+ fixed term (regardless of how the transaction is characterized), the
323	+ Corresponding Source conveyed under this section must be accompanied
324	+ by the Installation Information. But this requirement does not apply
325	+ if neither you nor any third party retains the ability to install
326	+ modified object code on the User Product (for example, the work has
327	+ been installed in ROM).
328	+
329	+ The requirement to provide Installation Information does not include a
330	+ requirement to continue to provide support service, warranty, or updates
331	+ for a work that has been modified or installed by the recipient, or for
332	+ the User Product in which it has been modified or installed. Access to a
333	+ network may be denied when the modification itself materially and
334	+ adversely affects the operation of the network or violates the rules and
335	+ protocols for communication across the network.
336	+
337	+ Corresponding Source conveyed, and Installation Information provided,
338	+ in accord with this section must be in a format that is publicly
339	+ documented (and with an implementation available to the public in
340	+ source code form), and must require no special password or key for
341	+ unpacking, reading or copying.
342	+
343	+ 7. Additional Terms.
344	+
345	+ "Additional permissions" are terms that supplement the terms of this
346	+ License by making exceptions from one or more of its conditions.
347	+ Additional permissions that are applicable to the entire Program shall
348	+ be treated as though they were included in this License, to the extent
349	+ that they are valid under applicable law. If additional permissions
350	+ apply only to part of the Program, that part may be used separately
351	+ under those permissions, but the entire Program remains governed by
352	+ this License without regard to the additional permissions.
353	+
354	+ When you convey a copy of a covered work, you may at your option
355	+ remove any additional permissions from that copy, or from any part of
356	+ it. (Additional permissions may be written to require their own
357	+ removal in certain cases when you modify the work.) You may place
358	+ additional permissions on material, added by you to a covered work,
359	+ for which you have or can give appropriate copyright permission.
360	+
361	+ Notwithstanding any other provision of this License, for material you
362	+ add to a covered work, you may (if authorized by the copyright holders of
363	+ that material) supplement the terms of this License with terms:
364	+
365	+ a) Disclaiming warranty or limiting liability differently from the
366	+ terms of sections 15 and 16 of this License; or
367	+
368	+ b) Requiring preservation of specified reasonable legal notices or
369	+ author attributions in that material or in the Appropriate Legal
370	+ Notices displayed by works containing it; or
371	+
372	+ c) Prohibiting misrepresentation of the origin of that material, or
373	+ requiring that modified versions of such material be marked in
374	+ reasonable ways as different from the original version; or
375	+
376	+ d) Limiting the use for publicity purposes of names of licensors or
377	+ authors of the material; or
378	+
379	+ e) Declining to grant rights under trademark law for use of some
380	+ trade names, trademarks, or service marks; or
381	+
382	+ f) Requiring indemnification of licensors and authors of that
383	+ material by anyone who conveys the material (or modified versions of
384	+ it) with contractual assumptions of liability to the recipient, for
385	+ any liability that these contractual assumptions directly impose on
386	+ those licensors and authors.
387	+
388	+ All other non-permissive additional terms are considered "further
389	+ restrictions" within the meaning of section 10. If the Program as you
390	+ received it, or any part of it, contains a notice stating that it is
391	+ governed by this License along with a term that is a further
392	+ restriction, you may remove that term. If a license document contains
393	+ a further restriction but permits relicensing or conveying under this
394	+ License, you may add to a covered work material governed by the terms
395	+ of that license document, provided that the further restriction does
396	+ not survive such relicensing or conveying.
397	+
398	+ If you add terms to a covered work in accord with this section, you
399	+ must place, in the relevant source files, a statement of the
400	+ additional terms that apply to those files, or a notice indicating
401	+ where to find the applicable terms.
402	+
403	+ Additional terms, permissive or non-permissive, may be stated in the
404	+ form of a separately written license, or stated as exceptions;
405	+ the above requirements apply either way.
406	+
407	+ 8. Termination.
408	+
409	+ You may not propagate or modify a covered work except as expressly
410	+ provided under this License. Any attempt otherwise to propagate or
411	+ modify it is void, and will automatically terminate your rights under
412	+ this License (including any patent licenses granted under the third
413	+ paragraph of section 11).
414	+
415	+ However, if you cease all violation of this License, then your
416	+ license from a particular copyright holder is reinstated (a)
417	+ provisionally, unless and until the copyright holder explicitly and
418	+ finally terminates your license, and (b) permanently, if the copyright
419	+ holder fails to notify you of the violation by some reasonable means
420	+ prior to 60 days after the cessation.
421	+
422	+ Moreover, your license from a particular copyright holder is
423	+ reinstated permanently if the copyright holder notifies you of the
424	+ violation by some reasonable means, this is the first time you have
425	+ received notice of violation of this License (for any work) from that
426	+ copyright holder, and you cure the violation prior to 30 days after
427	+ your receipt of the notice.
428	+
429	+ Termination of your rights under this section does not terminate the
430	+ licenses of parties who have received copies or rights from you under
431	+ this License. If your rights have been terminated and not permanently
432	+ reinstated, you do not qualify to receive new licenses for the same
433	+ material under section 10.
434	+
435	+ 9. Acceptance Not Required for Having Copies.
436	+
437	+ You are not required to accept this License in order to receive or
438	+ run a copy of the Program. Ancillary propagation of a covered work
439	+ occurring solely as a consequence of using peer-to-peer transmission
440	+ to receive a copy likewise does not require acceptance. However,
441	+ nothing other than this License grants you permission to propagate or
442	+ modify any covered work. These actions infringe copyright if you do
443	+ not accept this License. Therefore, by modifying or propagating a
444	+ covered work, you indicate your acceptance of this License to do so.
445	+
446	+ 10. Automatic Licensing of Downstream Recipients.
447	+
448	+ Each time you convey a covered work, the recipient automatically
449	+ receives a license from the original licensors, to run, modify and
450	+ propagate that work, subject to this License. You are not responsible
451	+ for enforcing compliance by third parties with this License.
452	+
453	+ An "entity transaction" is a transaction transferring control of an
454	+ organization, or substantially all assets of one, or subdividing an
455	+ organization, or merging organizations. If propagation of a covered
456	+ work results from an entity transaction, each party to that
457	+ transaction who receives a copy of the work also receives whatever
458	+ licenses to the work the party's predecessor in interest had or could
459	+ give under the previous paragraph, plus a right to possession of the
460	+ Corresponding Source of the work from the predecessor in interest, if
461	+ the predecessor has it or can get it with reasonable efforts.
462	+
463	+ You may not impose any further restrictions on the exercise of the
464	+ rights granted or affirmed under this License. For example, you may
465	+ not impose a license fee, royalty, or other charge for exercise of
466	+ rights granted under this License, and you may not initiate litigation
467	+ (including a cross-claim or counterclaim in a lawsuit) alleging that
468	+ any patent claim is infringed by making, using, selling, offering for
469	+ sale, or importing the Program or any portion of it.
470	+
471	+ 11. Patents.
472	+
473	+ A "contributor" is a copyright holder who authorizes use under this
474	+ License of the Program or a work on which the Program is based. The
475	+ work thus licensed is called the contributor's "contributor version".
476	+
477	+ A contributor's "essential patent claims" are all patent claims
478	+ owned or controlled by the contributor, whether already acquired or
479	+ hereafter acquired, that would be infringed by some manner, permitted
480	+ by this License, of making, using, or selling its contributor version,
481	+ but do not include claims that would be infringed only as a
482	+ consequence of further modification of the contributor version. For
483	+ purposes of this definition, "control" includes the right to grant
484	+ patent sublicenses in a manner consistent with the requirements of
485	+ this License.
486	+
487	+ Each contributor grants you a non-exclusive, worldwide, royalty-free
488	+ patent license under the contributor's essential patent claims, to
489	+ make, use, sell, offer for sale, import and otherwise run, modify and
490	+ propagate the contents of its contributor version.
491	+
492	+ In the following three paragraphs, a "patent license" is any express
493	+ agreement or commitment, however denominated, not to enforce a patent
494	+ (such as an express permission to practice a patent or covenant not to
495	+ sue for patent infringement). To "grant" such a patent license to a
496	+ party means to make such an agreement or commitment not to enforce a
497	+ patent against the party.
498	+
499	+ If you convey a covered work, knowingly relying on a patent license,
500	+ and the Corresponding Source of the work is not available for anyone
501	+ to copy, free of charge and under the terms of this License, through a
502	+ publicly available network server or other readily accessible means,
503	+ then you must either (1) cause the Corresponding Source to be so
504	+ available, or (2) arrange to deprive yourself of the benefit of the
505	+ patent license for this particular work, or (3) arrange, in a manner
506	+ consistent with the requirements of this License, to extend the patent
507	+ license to downstream recipients. "Knowingly relying" means you have
508	+ actual knowledge that, but for the patent license, your conveying the
509	+ covered work in a country, or your recipient's use of the covered work
510	+ in a country, would infringe one or more identifiable patents in that
511	+ country that you have reason to believe are valid.
512	+
513	+ If, pursuant to or in connection with a single transaction or
514	+ arrangement, you convey, or propagate by procuring conveyance of, a
515	+ covered work, and grant a patent license to some of the parties
516	+ receiving the covered work authorizing them to use, propagate, modify
517	+ or convey a specific copy of the covered work, then the patent license
518	+ you grant is automatically extended to all recipients of the covered
519	+ work and works based on it.
520	+
521	+ A patent license is "discriminatory" if it does not include within
522	+ the scope of its coverage, prohibits the exercise of, or is
523	+ conditioned on the non-exercise of one or more of the rights that are
524	+ specifically granted under this License. You may not convey a covered
525	+ work if you are a party to an arrangement with a third party that is
526	+ in the business of distributing software, under which you make payment
527	+ to the third party based on the extent of your activity of conveying
528	+ the work, and under which the third party grants, to any of the
529	+ parties who would receive the covered work from you, a discriminatory
530	+ patent license (a) in connection with copies of the covered work
531	+ conveyed by you (or copies made from those copies), or (b) primarily
532	+ for and in connection with specific products or compilations that
533	+ contain the covered work, unless you entered into that arrangement,
534	+ or that patent license was granted, prior to 28 March 2007.
535	+
536	+ Nothing in this License shall be construed as excluding or limiting
537	+ any implied license or other defenses to infringement that may
538	+ otherwise be available to you under applicable patent law.
539	+
540	+ 12. No Surrender of Others' Freedom.
541	+
542	+ If conditions are imposed on you (whether by court order, agreement or
543	+ otherwise) that contradict the conditions of this License, they do not
544	+ excuse you from the conditions of this License. If you cannot convey a
545	+ covered work so as to satisfy simultaneously your obligations under this
546	+ License and any other pertinent obligations, then as a consequence you may
547	+ not convey it at all. For example, if you agree to terms that obligate you
548	+ to collect a royalty for further conveying from those to whom you convey
549	+ the Program, the only way you could satisfy both those terms and this
550	+ License would be to refrain entirely from conveying the Program.
551	+
552	+ 13. Use with the GNU Affero General Public License.
553	+
554	+ Notwithstanding any other provision of this License, you have
555	+ permission to link or combine any covered work with a work licensed
556	+ under version 3 of the GNU Affero General Public License into a single
557	+ combined work, and to convey the resulting work. The terms of this
558	+ License will continue to apply to the part which is the covered work,
559	+ but the special requirements of the GNU Affero General Public License,
560	+ section 13, concerning interaction through a network will apply to the
561	+ combination as such.
562	+
563	+ 14. Revised Versions of this License.
564	+
565	+ The Free Software Foundation may publish revised and/or new versions of
566	+ the GNU General Public License from time to time. Such new versions will
567	+ be similar in spirit to the present version, but may differ in detail to
568	+ address new problems or concerns.
569	+
570	+ Each version is given a distinguishing version number. If the
571	+ Program specifies that a certain numbered version of the GNU General
572	+ Public License "or any later version" applies to it, you have the
573	+ option of following the terms and conditions either of that numbered
574	+ version or of any later version published by the Free Software
575	+ Foundation. If the Program does not specify a version number of the
576	+ GNU General Public License, you may choose any version ever published
577	+ by the Free Software Foundation.
578	+
579	+ If the Program specifies that a proxy can decide which future
580	+ versions of the GNU General Public License can be used, that proxy's
581	+ public statement of acceptance of a version permanently authorizes you
582	+ to choose that version for the Program.
583	+
584	+ Later license versions may give you additional or different
585	+ permissions. However, no additional obligations are imposed on any
586	+ author or copyright holder as a result of your choosing to follow a
587	+ later version.
588	+
589	+ 15. Disclaimer of Warranty.
590	+
591	+ THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
592	+ APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
593	+ HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
594	+ OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
595	+ THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
596	+ PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
597	+ IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
598	+ ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
599	+
600	+ 16. Limitation of Liability.
601	+
602	+ IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
603	+ WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
604	+ THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
605	+ GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
606	+ USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
607	+ DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
608	+ PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
609	+ EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
610	+ SUCH DAMAGES.
611	+
612	+ 17. Interpretation of Sections 15 and 16.
613	+
614	+ If the disclaimer of warranty and limitation of liability provided
615	+ above cannot be given local legal effect according to their terms,
616	+ reviewing courts shall apply local law that most closely approximates
617	+ an absolute waiver of all civil liability in connection with the
618	+ Program, unless a warranty or assumption of liability accompanies a
619	+ copy of the Program in return for a fee.
620	+
621	+ END OF TERMS AND CONDITIONS
622	+
623	+ How to Apply These Terms to Your New Programs
624	+
625	+ If you develop a new program, and you want it to be of the greatest
626	+ possible use to the public, the best way to achieve this is to make it
627	+ free software which everyone can redistribute and change under these terms.
628	+
629	+ To do so, attach the following notices to the program. It is safest
630	+ to attach them to the start of each source file to most effectively
631	+ state the exclusion of warranty; and each file should have at least
632	+ the "copyright" line and a pointer to where the full notice is found.
633	+
634	+ {one line to give the program's name and a brief idea of what it does.}
635	+ Copyright (C) {year} {name of author}
636	+
637	+ This program is free software: you can redistribute it and/or modify
638	+ it under the terms of the GNU General Public License as published by
639	+ the Free Software Foundation, either version 3 of the License, or
640	+ (at your option) any later version.
641	+
642	+ This program is distributed in the hope that it will be useful,
643	+ but WITHOUT ANY WARRANTY; without even the implied warranty of
644	+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
645	+ GNU General Public License for more details.
646	+
647	+ You should have received a copy of the GNU General Public License
648	+ along with this program. If not, see <http://www.gnu.org/licenses/>.
649	+
650	+ Also add information on how to contact you by electronic and paper mail.
651	+
652	+ If the program does terminal interaction, make it output a short
653	+ notice like this when it starts in an interactive mode:
654	+
655	+ Matomo Copyright (C) 2007-2018 Matomo.org
656	+
657	+ This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
658	+ This is free software, and you are welcome to redistribute it
659	+ under certain conditions; type `show c' for details.
660	+
661	+ The hypothetical commands `show w' and `show c' should show the appropriate
662	+ parts of the General Public License. Of course, your program's commands
663	+ might be different; for a GUI interface, you would use an "about box".
664	+
665	+ You should also get your employer (if you work as a programmer) or school,
666	+ if any, to sign a "copyright disclaimer" for the program, if necessary.
667	+ For more information on this, and how to apply and follow the GNU GPL, see
668	+ <http://www.gnu.org/licenses/>.
669	+
670	+ The GNU General Public License does not permit incorporating your program
671	+ into proprietary programs. If your program is a subroutine library, you
672	+ may consider it more useful to permit linking proprietary applications with
673	+ the library. If this is what you want to do, use the GNU Lesser General
674	+ Public License instead of this License. But first, please read
675	+ <http://www.gnu.org/philosophy/why-not-lgpl.html>.

app/.htaccess ADDED Viewed

	@@ -0,0 +1,5 @@


1	+ # Serve HTML files as text/html mime type - Note: requires mod_mime apache module!
2	+ <IfModule mod_mime.c>
3	+ AddHandler text/html .html
4	+ AddHandler text/html .htm
5	+ </IfModule>

app/LEGALNOTICE ADDED Viewed

	@@ -0,0 +1,295 @@


1	+ COPYRIGHT
2	+
3	+ Matomo - free/libre analytics platform
4	+
5	+ The software package is:
6	+
7	+ Copyright (C) 2014 Matthieu Aubry
8	+
9	+ Individual contributions, components, and libraries are copyright
10	+ of their respective authors.
11	+
12	+
13	+ SOFTWARE LICENSE
14	+
15	+ The free software license of Matomo is GNU General Public License v3
16	+ or later. A copy of GNU GPL v3 should have been included in this
17	+ software package in LICENSE.
18	+
19	+
20	+ TRADEMARK
21	+
22	+ Matomo (TM) is an internationally registered trademark.
23	+
24	+ The software license does not grant any rights under trademark
25	+ law for use of the trademark. Refer to https://matomo.org/trademark/
26	+ for up-to-date trademark licensing information.
27	+
28	+ *
29	+ * The software license applies to both the aggregate software and
30	+ * application-specific portions of the software.
31	+ *
32	+ * You may not remove this legal notice or modify the software
33	+ * in such a way that misrepresents the origin of the software.
34	+ *
35	+
36	+ CREDITS
37	+
38	+ The software consists of contributions made by many individuals.
39	+ Major contributors are listed in https://matomo.org/team/.
40	+
41	+ For detailed contribution history, refer to the source, tickets,
42	+ patches, and Git revision history, available at
43	+ https://github.com/matomo-org/matomo/issues
44	+ https://github.com/matomo-org/matomo
45	+
46	+
47	+ SEPARATELY LICENSED COMPONENTS AND LIBRARIES
48	+
49	+ The following components/libraries are distributed in this package,
50	+ and subject to their respective licenses.
51	+
52	+ Name: javascriptCode.tpl - tracking tag to embed in your web pages
53	+ Link: https://github.com/matomo-org/matomo/blob/master/core/Tracker/javascriptTag.tpl
54	+ License: Public Domain
55	+
56	+ Name: jquery.truncate
57	+ Link: https://github.com/matomo-org/matomo/blob/master/libs/jquery/truncate/
58	+ License: New BSD
59	+
60	+ Name: matomo.js & piwik.js - JavaScript tracker
61	+ Link: https://github.com/matomo-org/matomo/blob/master/js/piwik.js
62	+ Link: https://github.com/matomo-org/matomo/blob/master/js/matomo.js
63	+ License: New BSD
64	+
65	+ Name: PiwikTracker - server-side tracker (PHP)
66	+ Link: https://github.com/matomo-org/matomo/blob/master/libs/PiwikTracker/
67	+ License: New BSD
68	+
69	+ Name: DeviceDetector
70	+ Link: https://github.com/matomo-org/device-detector
71	+ License: LGPL
72	+
73	+ Name: Piwik/Decompress
74	+ Link: https://github.com/matomo-org/component-decompress
75	+ License: LGPL v3.0
76	+
77	+ Name: Piwik/Network
78	+ Link: https://github.com/matomo-org/component-network
79	+ License: LGPL v3.0
80	+
81	+
82	+ THIRD-PARTY COMPONENTS AND LIBRARIES
83	+
84	+ The following components/libraries are redistributed in this package,
85	+ and subject to their respective licenses.
86	+
87	+ Name: jqPlot
88	+ Link: http://www.jqplot.com/
89	+ License: Dual-licensed: MIT (Expat) or GPL v2
90	+
91	+ Name: jQuery
92	+ Link: https://jquery.com/
93	+ License: Dual-licensed: MIT (Expat) or GPL
94	+ Notes:
95	+ - GPL version not explicitly stated in source but GPL v2 is in git
96	+ - includes Sizzle.js - multi-licensed: MIT (Expat), New BSD, or GPL [v2]
97	+
98	+ Name: jQuery UI
99	+ Link: https://jqueryui.com/
100	+ License: Dual-licensed: MIT (Expat) or GPL
101	+ Notes:
102	+ - GPL version not explicitly stated in source but GPL v2 is in git
103	+
104	+ Name: jquery.history
105	+ Link: https://tkyk.github.io/jquery-history-plugin/
106	+ License: MIT (Expat)
107	+
108	+ Name: jquery.scrollTo
109	+ Link: http://plugins.jquery.com/project/ScrollTo
110	+ License: Dual licensed: MIT (Expat) or GPL
111	+
112	+ Name: jquery Tooltip
113	+ Link: http://bassistance.de/jquery-plugins/jquery-plugin-tooltip/
114	+ License: Dual licensed: MIT (Expat) or GPL
115	+
116	+ Name: jquery placeholder
117	+ Link: http://mths.be/placeholder
118	+ License: Dual licensed: MIT (Expat) or GPL
119	+
120	+ Name: qrcode.js
121	+ Link: https://github.com/davidshimjs/qrcodejs
122	+ License: MIT
123	+
124	+ Name: json2.js
125	+ Link: http://json.org/
126	+ License: Public domain
127	+ Notes:
128	+ - reference implementation
129	+
130	+ Name: jshrink
131	+ Link: https://github.com/tedivm/jshrink
132	+ License: BSD-3-Clause
133	+
134	+ Name: Sparkline
135	+ Link: https://github.com/jamiebicknell/Sparkline
136	+ License: MIT
137	+
138	+ Name: sprintf
139	+ Link: http://www.diveintojavascript.com/projects/javascript-sprintf
140	+ License: New BSD
141	+
142	+ Name: upgrade.php
143	+ Link: http://upgradephp.berlios.de/
144	+ License: Public domain
145	+
146	+ Name: Archive Tar
147	+ Link: https://pear.php.net/package/Archive_Tar
148	+ License: New BSD
149	+
150	+ Name: Event Dispatcher (and Notification)
151	+ Link: https://pear.php.net/package/Event_Dispatcher/
152	+ License: New BSD
153	+
154	+ Name: HTML Common2
155	+ Link: https://pear.php.net/package/HTML_Common2/
156	+ License: New BSD
157	+
158	+ Name: HTML QuickForm2
159	+ Link: https://pear.php.net/package/HTML_QuickForm2/
160	+ License: New BSD
161	+
162	+ Name: HTML QuickForm2_Renderer_Smarty
163	+ Link: http://www.phcomp.co.uk/tmp/Smarty.phps
164	+ License: New BSD
165	+
166	+ Name: MaxMindGeoIP
167	+ Link: https://dev.maxmind.com/geoip/legacy/downloadable/#PHP-7
168	+ License: LGPL
169	+
170	+ Name: PclZip
171	+ Link: http://www.phpconcept.net/pclzip/
172	+ License: LGPL
173	+ Notes:
174	+ - GPL version not explicitly stated but tarball contains LGPL v2.1
175	+
176	+ Name: PEAR (base system)
177	+ Link: https://pear.php.net/package/PEAR
178	+ License: New BSD
179	+
180	+ Name: PhpSecInfo
181	+ Link: http://phpsec.org/projects/phpsecinfo/
182	+ License: New BSD
183	+
184	+ Name: RankChecker
185	+ Link: http://www.getrank.org/free-pagerank-script
186	+ License: GPL
187	+
188	+ Name: Twig
189	+ Link: https://twig.symfony.com/
190	+ License: BSD
191	+
192	+ Name: TCPDF
193	+ Link: https://sourceforge.net/projects/tcpdf/
194	+ License: LGPL v3 or later
195	+
196	+ Name: Zend Framework
197	+ Link: https://framework.zend.com/
198	+ License: New BSD
199	+
200	+ Name: pChart 2.1.4
201	+ Link: http://www.pchart.net
202	+ License: GPL v3
203	+
204	+ Name: Chroma.js
205	+ Link: https://github.com/gka/chroma.js
206	+ License: GPL v3
207	+
208	+ Name: qTip2 - Pretty powerful tooltips
209	+ Link: http://craigsworks.com/projects/qtip2/
210	+ License: GPL
211	+
212	+ Name: Kartograph.js
213	+ Link: http://kartograph.org/
214	+ License: LGPL v3 or later
215	+
216	+ Name: Raphaël - JavaScript Vector Library
217	+ Link: http://raphaeljs.com/
218	+ License: MIT (Expat)
219	+
220	+ Name: iFrame Resizer
221	+ Link: https://github.com/davidjbradshaw/iframe-resizer
222	+ License: MIT
223	+
224	+ Name: lessphp
225	+ Link: http://leafo.net/lessphp
226	+ License: GPL3, MIT (Expat)
227	+
228	+ Name: Symfony Console Component
229	+ Link: https://github.com/symfony/Console
230	+ License: MIT (Expat)
231	+
232	+ Name: AngularJS
233	+ Link: https://github.com/angular/angular.js
234	+ License: MIT (Expat)
235	+
236	+ Name: Mousetrap
237	+ Link: https://github.com/ccampbell/mousetrap
238	+ License: Apache 2.0
239	+
240	+ Name: PHP-DI
241	+ Link: http://php-di.org/
242	+ License: MIT (Expat)
243	+
244	+
245	+ THIRD-PARTY CONTENT
246	+
247	+ Name: FamFamFam icons - Mark James
248	+ Link: http://www.famfamfam.com/lab/icons/
249	+ License: CC BY 3.0
250	+
251	+ Name: Solar System icons - Dan Wiersema
252	+ Link: http://www.iconspedia.com/icon/neptune-4672.html
253	+ License: Free for non-commercial use
254	+ Notes:
255	+ - used in Matomo's ExampleUI plugin
256	+
257	+ Name: flag-icon-css - Lipis
258	+ Link: https://github.com/lipis/flag-icon-css
259	+ License: MIT (Expat)
260	+ Notes:
261	+ - used for flag PNGs
262	+
263	+ Name: Wine project - tahoma.ttf font
264	+ Link: http://source.winehq.org/git/wine.git/blob_plain/HEAD:/fonts/tahoma.ttf
265	+ License: LGPL v2.1
266	+ Notes:
267	+ - used in ImageGraph plugin
268	+
269	+ Name: plugins/Feedback/angularjs/ratefeature/thumbs-down.png
270	+ Link: https://www.iconfinder.com/icons/216428/down_thumbs_icon
271	+ License: Creative Commons (Attribution-Share Alike 3.0 Unported)
272	+
273	+ Name: plugins/Feedback/angularjs/ratefeature/thumbs-up.png
274	+ Link: https://www.iconfinder.com/icons/216429/thumbs_up_icon
275	+ License: Creative Commons (Attribution-Share Alike 3.0 Unported)
276	+
277	+ Name: plugins/Insights/images/idea.png
278	+ Link: https://www.iconfinder.com/icons/6074/brainstorm_bulb_idea_jabber_light_icon
279	+ License: GPL
280	+ By: Alessandro Rei - http://www.kde-look.org/usermanager/search.php?username=mentalrey
281	+
282	+ Name: Material icons ("icon-info2", "icon-outline", "icon-settings", "icon-form", "icon-play", "icon-pause", "icon-replay", "icon-skip-next", "icon-skip-forward", "icon-stop", "icon-fast-forward", "icon-fast-rewind", "icon-bug", "icon-upload", "icon-segmented-visits-log") in plugins/Morpheus/fonts, and plugins/Morpheus/images/compare.svg
283	+ Link: https://design.google.com/icons/
284	+ License: Apache License Version 2.0
285	+
286	+ Name: IcoMoon - Free icons ("icon-funnel", "icon-lab", "icon-archive", "icon-rocket", "icon-embed") in plugins/Morpheus/fonts
287	+ Link: https://icomoon.io/#icons-icomoon
288	+ License: GPL
289	+
290	+ Notes:
291	+ - the "New BSD" license refers to either the "Modified BSD" and "Simplified BSD"
292	+ licenses (2- or 3-clause), which are GPL compatible.
293	+ - icons for browsers, operating systems, browser plugins, brands, search engines, social media websites
294	+ and flags of countries are nominative use of third-party trademarks when
295	+ referring to the corresponding product or entity

app/LICENSE ADDED Viewed

	@@ -0,0 +1,675 @@


1	+ GNU GENERAL PUBLIC LICENSE
2	+ Version 3, 29 June 2007
3	+
4	+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
5	+ Everyone is permitted to copy and distribute verbatim copies
6	+ of this license document, but changing it is not allowed.
7	+
8	+ Preamble
9	+
10	+ The GNU General Public License is a free, copyleft license for
11	+ software and other kinds of works.
12	+
13	+ The licenses for most software and other practical works are designed
14	+ to take away your freedom to share and change the works. By contrast,
15	+ the GNU General Public License is intended to guarantee your freedom to
16	+ share and change all versions of a program--to make sure it remains free
17	+ software for all its users. We, the Free Software Foundation, use the
18	+ GNU General Public License for most of our software; it applies also to
19	+ any other work released this way by its authors. You can apply it to
20	+ your programs, too.
21	+
22	+ When we speak of free software, we are referring to freedom, not
23	+ price. Our General Public Licenses are designed to make sure that you
24	+ have the freedom to distribute copies of free software (and charge for
25	+ them if you wish), that you receive source code or can get it if you
26	+ want it, that you can change the software or use pieces of it in new
27	+ free programs, and that you know you can do these things.
28	+
29	+ To protect your rights, we need to prevent others from denying you
30	+ these rights or asking you to surrender the rights. Therefore, you have
31	+ certain responsibilities if you distribute copies of the software, or if
32	+ you modify it: responsibilities to respect the freedom of others.
33	+
34	+ For example, if you distribute copies of such a program, whether
35	+ gratis or for a fee, you must pass on to the recipients the same
36	+ freedoms that you received. You must make sure that they, too, receive
37	+ or can get the source code. And you must show them these terms so they
38	+ know their rights.
39	+
40	+ Developers that use the GNU GPL protect your rights with two steps:
41	+ (1) assert copyright on the software, and (2) offer you this License
42	+ giving you legal permission to copy, distribute and/or modify it.
43	+
44	+ For the developers' and authors' protection, the GPL clearly explains
45	+ that there is no warranty for this free software. For both users' and
46	+ authors' sake, the GPL requires that modified versions be marked as
47	+ changed, so that their problems will not be attributed erroneously to
48	+ authors of previous versions.
49	+
50	+ Some devices are designed to deny users access to install or run
51	+ modified versions of the software inside them, although the manufacturer
52	+ can do so. This is fundamentally incompatible with the aim of
53	+ protecting users' freedom to change the software. The systematic
54	+ pattern of such abuse occurs in the area of products for individuals to
55	+ use, which is precisely where it is most unacceptable. Therefore, we
56	+ have designed this version of the GPL to prohibit the practice for those
57	+ products. If such problems arise substantially in other domains, we
58	+ stand ready to extend this provision to those domains in future versions
59	+ of the GPL, as needed to protect the freedom of users.
60	+
61	+ Finally, every program is threatened constantly by software patents.
62	+ States should not allow patents to restrict development and use of
63	+ software on general-purpose computers, but in those that do, we wish to
64	+ avoid the special danger that patents applied to a free program could
65	+ make it effectively proprietary. To prevent this, the GPL assures that
66	+ patents cannot be used to render the program non-free.
67	+
68	+ The precise terms and conditions for copying, distribution and
69	+ modification follow.
70	+
71	+ TERMS AND CONDITIONS
72	+
73	+ 0. Definitions.
74	+
75	+ "This License" refers to version 3 of the GNU General Public License.
76	+
77	+ "Copyright" also means copyright-like laws that apply to other kinds of
78	+ works, such as semiconductor masks.
79	+
80	+ "The Program" refers to any copyrightable work licensed under this
81	+ License. Each licensee is addressed as "you". "Licensees" and
82	+ "recipients" may be individuals or organizations.
83	+
84	+ To "modify" a work means to copy from or adapt all or part of the work
85	+ in a fashion requiring copyright permission, other than the making of an
86	+ exact copy. The resulting work is called a "modified version" of the
87	+ earlier work or a work "based on" the earlier work.
88	+
89	+ A "covered work" means either the unmodified Program or a work based
90	+ on the Program.
91	+
92	+ To "propagate" a work means to do anything with it that, without
93	+ permission, would make you directly or secondarily liable for
94	+ infringement under applicable copyright law, except executing it on a
95	+ computer or modifying a private copy. Propagation includes copying,
96	+ distribution (with or without modification), making available to the
97	+ public, and in some countries other activities as well.
98	+
99	+ To "convey" a work means any kind of propagation that enables other
100	+ parties to make or receive copies. Mere interaction with a user through
101	+ a computer network, with no transfer of a copy, is not conveying.
102	+
103	+ An interactive user interface displays "Appropriate Legal Notices"
104	+ to the extent that it includes a convenient and prominently visible
105	+ feature that (1) displays an appropriate copyright notice, and (2)
106	+ tells the user that there is no warranty for the work (except to the
107	+ extent that warranties are provided), that licensees may convey the
108	+ work under this License, and how to view a copy of this License. If
109	+ the interface presents a list of user commands or options, such as a
110	+ menu, a prominent item in the list meets this criterion.
111	+
112	+ 1. Source Code.
113	+
114	+ The "source code" for a work means the preferred form of the work
115	+ for making modifications to it. "Object code" means any non-source
116	+ form of a work.
117	+
118	+ A "Standard Interface" means an interface that either is an official
119	+ standard defined by a recognized standards body, or, in the case of
120	+ interfaces specified for a particular programming language, one that
121	+ is widely used among developers working in that language.
122	+
123	+ The "System Libraries" of an executable work include anything, other
124	+ than the work as a whole, that (a) is included in the normal form of
125	+ packaging a Major Component, but which is not part of that Major
126	+ Component, and (b) serves only to enable use of the work with that
127	+ Major Component, or to implement a Standard Interface for which an
128	+ implementation is available to the public in source code form. A
129	+ "Major Component", in this context, means a major essential component
130	+ (kernel, window system, and so on) of the specific operating system
131	+ (if any) on which the executable work runs, or a compiler used to
132	+ produce the work, or an object code interpreter used to run it.
133	+
134	+ The "Corresponding Source" for a work in object code form means all
135	+ the source code needed to generate, install, and (for an executable
136	+ work) run the object code and to modify the work, including scripts to
137	+ control those activities. However, it does not include the work's
138	+ System Libraries, or general-purpose tools or generally available free
139	+ programs which are used unmodified in performing those activities but
140	+ which are not part of the work. For example, Corresponding Source
141	+ includes interface definition files associated with source files for
142	+ the work, and the source code for shared libraries and dynamically
143	+ linked subprograms that the work is specifically designed to require,
144	+ such as by intimate data communication or control flow between those
145	+ subprograms and other parts of the work.
146	+
147	+ The Corresponding Source need not include anything that users
148	+ can regenerate automatically from other parts of the Corresponding
149	+ Source.
150	+
151	+ The Corresponding Source for a work in source code form is that
152	+ same work.
153	+
154	+ 2. Basic Permissions.
155	+
156	+ All rights granted under this License are granted for the term of
157	+ copyright on the Program, and are irrevocable provided the stated
158	+ conditions are met. This License explicitly affirms your unlimited
159	+ permission to run the unmodified Program. The output from running a
160	+ covered work is covered by this License only if the output, given its
161	+ content, constitutes a covered work. This License acknowledges your
162	+ rights of fair use or other equivalent, as provided by copyright law.
163	+
164	+ You may make, run and propagate covered works that you do not
165	+ convey, without conditions so long as your license otherwise remains
166	+ in force. You may convey covered works to others for the sole purpose
167	+ of having them make modifications exclusively for you, or provide you
168	+ with facilities for running those works, provided that you comply with
169	+ the terms of this License in conveying all material for which you do
170	+ not control copyright. Those thus making or running the covered works
171	+ for you must do so exclusively on your behalf, under your direction
172	+ and control, on terms that prohibit them from making any copies of
173	+ your copyrighted material outside their relationship with you.
174	+
175	+ Conveying under any other circumstances is permitted solely under
176	+ the conditions stated below. Sublicensing is not allowed; section 10
177	+ makes it unnecessary.
178	+
179	+ 3. Protecting Users' Legal Rights From Anti-Circumvention Law.
180	+
181	+ No covered work shall be deemed part of an effective technological
182	+ measure under any applicable law fulfilling obligations under article
183	+ 11 of the WIPO copyright treaty adopted on 20 December 1996, or
184	+ similar laws prohibiting or restricting circumvention of such
185	+ measures.
186	+
187	+ When you convey a covered work, you waive any legal power to forbid
188	+ circumvention of technological measures to the extent such circumvention
189	+ is effected by exercising rights under this License with respect to
190	+ the covered work, and you disclaim any intention to limit operation or
191	+ modification of the work as a means of enforcing, against the work's
192	+ users, your or third parties' legal rights to forbid circumvention of
193	+ technological measures.
194	+
195	+ 4. Conveying Verbatim Copies.
196	+
197	+ You may convey verbatim copies of the Program's source code as you
198	+ receive it, in any medium, provided that you conspicuously and
199	+ appropriately publish on each copy an appropriate copyright notice;
200	+ keep intact all notices stating that this License and any
201	+ non-permissive terms added in accord with section 7 apply to the code;
202	+ keep intact all notices of the absence of any warranty; and give all
203	+ recipients a copy of this License along with the Program.
204	+
205	+ You may charge any price or no price for each copy that you convey,
206	+ and you may offer support or warranty protection for a fee.
207	+
208	+ 5. Conveying Modified Source Versions.
209	+
210	+ You may convey a work based on the Program, or the modifications to
211	+ produce it from the Program, in the form of source code under the
212	+ terms of section 4, provided that you also meet all of these conditions:
213	+
214	+ a) The work must carry prominent notices stating that you modified
215	+ it, and giving a relevant date.
216	+
217	+ b) The work must carry prominent notices stating that it is
218	+ released under this License and any conditions added under section
219	+ 7. This requirement modifies the requirement in section 4 to
220	+ "keep intact all notices".
221	+
222	+ c) You must license the entire work, as a whole, under this
223	+ License to anyone who comes into possession of a copy. This
224	+ License will therefore apply, along with any applicable section 7
225	+ additional terms, to the whole of the work, and all its parts,
226	+ regardless of how they are packaged. This License gives no
227	+ permission to license the work in any other way, but it does not
228	+ invalidate such permission if you have separately received it.
229	+
230	+ d) If the work has interactive user interfaces, each must display
231	+ Appropriate Legal Notices; however, if the Program has interactive
232	+ interfaces that do not display Appropriate Legal Notices, your
233	+ work need not make them do so.
234	+
235	+ A compilation of a covered work with other separate and independent
236	+ works, which are not by their nature extensions of the covered work,
237	+ and which are not combined with it such as to form a larger program,
238	+ in or on a volume of a storage or distribution medium, is called an
239	+ "aggregate" if the compilation and its resulting copyright are not
240	+ used to limit the access or legal rights of the compilation's users
241	+ beyond what the individual works permit. Inclusion of a covered work
242	+ in an aggregate does not cause this License to apply to the other
243	+ parts of the aggregate.
244	+
245	+ 6. Conveying Non-Source Forms.
246	+
247	+ You may convey a covered work in object code form under the terms
248	+ of sections 4 and 5, provided that you also convey the
249	+ machine-readable Corresponding Source under the terms of this License,
250	+ in one of these ways:
251	+
252	+ a) Convey the object code in, or embodied in, a physical product
253	+ (including a physical distribution medium), accompanied by the
254	+ Corresponding Source fixed on a durable physical medium
255	+ customarily used for software interchange.
256	+
257	+ b) Convey the object code in, or embodied in, a physical product
258	+ (including a physical distribution medium), accompanied by a
259	+ written offer, valid for at least three years and valid for as
260	+ long as you offer spare parts or customer support for that product
261	+ model, to give anyone who possesses the object code either (1) a
262	+ copy of the Corresponding Source for all the software in the
263	+ product that is covered by this License, on a durable physical
264	+ medium customarily used for software interchange, for a price no
265	+ more than your reasonable cost of physically performing this
266	+ conveying of source, or (2) access to copy the
267	+ Corresponding Source from a network server at no charge.
268	+
269	+ c) Convey individual copies of the object code with a copy of the
270	+ written offer to provide the Corresponding Source. This
271	+ alternative is allowed only occasionally and noncommercially, and
272	+ only if you received the object code with such an offer, in accord
273	+ with subsection 6b.
274	+
275	+ d) Convey the object code by offering access from a designated
276	+ place (gratis or for a charge), and offer equivalent access to the
277	+ Corresponding Source in the same way through the same place at no
278	+ further charge. You need not require recipients to copy the
279	+ Corresponding Source along with the object code. If the place to
280	+ copy the object code is a network server, the Corresponding Source
281	+ may be on a different server (operated by you or a third party)
282	+ that supports equivalent copying facilities, provided you maintain
283	+ clear directions next to the object code saying where to find the
284	+ Corresponding Source. Regardless of what server hosts the
285	+ Corresponding Source, you remain obligated to ensure that it is
286	+ available for as long as needed to satisfy these requirements.
287	+
288	+ e) Convey the object code using peer-to-peer transmission, provided
289	+ you inform other peers where the object code and Corresponding
290	+ Source of the work are being offered to the general public at no
291	+ charge under subsection 6d.
292	+
293	+ A separable portion of the object code, whose source code is excluded
294	+ from the Corresponding Source as a System Library, need not be
295	+ included in conveying the object code work.
296	+
297	+ A "User Product" is either (1) a "consumer product", which means any
298	+ tangible personal property which is normally used for personal, family,
299	+ or household purposes, or (2) anything designed or sold for incorporation
300	+ into a dwelling. In determining whether a product is a consumer product,
301	+ doubtful cases shall be resolved in favor of coverage. For a particular
302	+ product received by a particular user, "normally used" refers to a
303	+ typical or common use of that class of product, regardless of the status
304	+ of the particular user or of the way in which the particular user
305	+ actually uses, or expects or is expected to use, the product. A product
306	+ is a consumer product regardless of whether the product has substantial
307	+ commercial, industrial or non-consumer uses, unless such uses represent
308	+ the only significant mode of use of the product.
309	+
310	+ "Installation Information" for a User Product means any methods,
311	+ procedures, authorization keys, or other information required to install
312	+ and execute modified versions of a covered work in that User Product from
313	+ a modified version of its Corresponding Source. The information must
314	+ suffice to ensure that the continued functioning of the modified object
315	+ code is in no case prevented or interfered with solely because
316	+ modification has been made.
317	+
318	+ If you convey an object code work under this section in, or with, or
319	+ specifically for use in, a User Product, and the conveying occurs as
320	+ part of a transaction in which the right of possession and use of the
321	+ User Product is transferred to the recipient in perpetuity or for a
322	+ fixed term (regardless of how the transaction is characterized), the
323	+ Corresponding Source conveyed under this section must be accompanied
324	+ by the Installation Information. But this requirement does not apply
325	+ if neither you nor any third party retains the ability to install
326	+ modified object code on the User Product (for example, the work has
327	+ been installed in ROM).
328	+
329	+ The requirement to provide Installation Information does not include a
330	+ requirement to continue to provide support service, warranty, or updates
331	+ for a work that has been modified or installed by the recipient, or for
332	+ the User Product in which it has been modified or installed. Access to a
333	+ network may be denied when the modification itself materially and
334	+ adversely affects the operation of the network or violates the rules and
335	+ protocols for communication across the network.
336	+
337	+ Corresponding Source conveyed, and Installation Information provided,
338	+ in accord with this section must be in a format that is publicly
339	+ documented (and with an implementation available to the public in
340	+ source code form), and must require no special password or key for
341	+ unpacking, reading or copying.
342	+
343	+ 7. Additional Terms.
344	+
345	+ "Additional permissions" are terms that supplement the terms of this
346	+ License by making exceptions from one or more of its conditions.
347	+ Additional permissions that are applicable to the entire Program shall
348	+ be treated as though they were included in this License, to the extent
349	+ that they are valid under applicable law. If additional permissions
350	+ apply only to part of the Program, that part may be used separately
351	+ under those permissions, but the entire Program remains governed by
352	+ this License without regard to the additional permissions.
353	+
354	+ When you convey a copy of a covered work, you may at your option
355	+ remove any additional permissions from that copy, or from any part of
356	+ it. (Additional permissions may be written to require their own
357	+ removal in certain cases when you modify the work.) You may place
358	+ additional permissions on material, added by you to a covered work,
359	+ for which you have or can give appropriate copyright permission.
360	+
361	+ Notwithstanding any other provision of this License, for material you
362	+ add to a covered work, you may (if authorized by the copyright holders of
363	+ that material) supplement the terms of this License with terms:
364	+
365	+ a) Disclaiming warranty or limiting liability differently from the
366	+ terms of sections 15 and 16 of this License; or
367	+
368	+ b) Requiring preservation of specified reasonable legal notices or
369	+ author attributions in that material or in the Appropriate Legal
370	+ Notices displayed by works containing it; or
371	+
372	+ c) Prohibiting misrepresentation of the origin of that material, or
373	+ requiring that modified versions of such material be marked in
374	+ reasonable ways as different from the original version; or
375	+
376	+ d) Limiting the use for publicity purposes of names of licensors or
377	+ authors of the material; or
378	+
379	+ e) Declining to grant rights under trademark law for use of some
380	+ trade names, trademarks, or service marks; or
381	+
382	+ f) Requiring indemnification of licensors and authors of that
383	+ material by anyone who conveys the material (or modified versions of
384	+ it) with contractual assumptions of liability to the recipient, for
385	+ any liability that these contractual assumptions directly impose on
386	+ those licensors and authors.
387	+
388	+ All other non-permissive additional terms are considered "further
389	+ restrictions" within the meaning of section 10. If the Program as you
390	+ received it, or any part of it, contains a notice stating that it is
391	+ governed by this License along with a term that is a further
392	+ restriction, you may remove that term. If a license document contains
393	+ a further restriction but permits relicensing or conveying under this
394	+ License, you may add to a covered work material governed by the terms
395	+ of that license document, provided that the further restriction does
396	+ not survive such relicensing or conveying.
397	+
398	+ If you add terms to a covered work in accord with this section, you
399	+ must place, in the relevant source files, a statement of the
400	+ additional terms that apply to those files, or a notice indicating
401	+ where to find the applicable terms.
402	+
403	+ Additional terms, permissive or non-permissive, may be stated in the
404	+ form of a separately written license, or stated as exceptions;
405	+ the above requirements apply either way.
406	+
407	+ 8. Termination.
408	+
409	+ You may not propagate or modify a covered work except as expressly
410	+ provided under this License. Any attempt otherwise to propagate or
411	+ modify it is void, and will automatically terminate your rights under
412	+ this License (including any patent licenses granted under the third
413	+ paragraph of section 11).
414	+
415	+ However, if you cease all violation of this License, then your
416	+ license from a particular copyright holder is reinstated (a)
417	+ provisionally, unless and until the copyright holder explicitly and
418	+ finally terminates your license, and (b) permanently, if the copyright
419	+ holder fails to notify you of the violation by some reasonable means
420	+ prior to 60 days after the cessation.
421	+
422	+ Moreover, your license from a particular copyright holder is
423	+ reinstated permanently if the copyright holder notifies you of the
424	+ violation by some reasonable means, this is the first time you have
425	+ received notice of violation of this License (for any work) from that
426	+ copyright holder, and you cure the violation prior to 30 days after
427	+ your receipt of the notice.
428	+
429	+ Termination of your rights under this section does not terminate the
430	+ licenses of parties who have received copies or rights from you under
431	+ this License. If your rights have been terminated and not permanently
432	+ reinstated, you do not qualify to receive new licenses for the same
433	+ material under section 10.
434	+
435	+ 9. Acceptance Not Required for Having Copies.
436	+
437	+ You are not required to accept this License in order to receive or
438	+ run a copy of the Program. Ancillary propagation of a covered work
439	+ occurring solely as a consequence of using peer-to-peer transmission
440	+ to receive a copy likewise does not require acceptance. However,
441	+ nothing other than this License grants you permission to propagate or
442	+ modify any covered work. These actions infringe copyright if you do
443	+ not accept this License. Therefore, by modifying or propagating a
444	+ covered work, you indicate your acceptance of this License to do so.
445	+
446	+ 10. Automatic Licensing of Downstream Recipients.
447	+
448	+ Each time you convey a covered work, the recipient automatically
449	+ receives a license from the original licensors, to run, modify and
450	+ propagate that work, subject to this License. You are not responsible
451	+ for enforcing compliance by third parties with this License.
452	+
453	+ An "entity transaction" is a transaction transferring control of an
454	+ organization, or substantially all assets of one, or subdividing an
455	+ organization, or merging organizations. If propagation of a covered
456	+ work results from an entity transaction, each party to that
457	+ transaction who receives a copy of the work also receives whatever
458	+ licenses to the work the party's predecessor in interest had or could
459	+ give under the previous paragraph, plus a right to possession of the
460	+ Corresponding Source of the work from the predecessor in interest, if
461	+ the predecessor has it or can get it with reasonable efforts.
462	+
463	+ You may not impose any further restrictions on the exercise of the
464	+ rights granted or affirmed under this License. For example, you may
465	+ not impose a license fee, royalty, or other charge for exercise of
466	+ rights granted under this License, and you may not initiate litigation
467	+ (including a cross-claim or counterclaim in a lawsuit) alleging that
468	+ any patent claim is infringed by making, using, selling, offering for
469	+ sale, or importing the Program or any portion of it.
470	+
471	+ 11. Patents.
472	+
473	+ A "contributor" is a copyright holder who authorizes use under this
474	+ License of the Program or a work on which the Program is based. The
475	+ work thus licensed is called the contributor's "contributor version".
476	+
477	+ A contributor's "essential patent claims" are all patent claims
478	+ owned or controlled by the contributor, whether already acquired or
479	+ hereafter acquired, that would be infringed by some manner, permitted
480	+ by this License, of making, using, or selling its contributor version,
481	+ but do not include claims that would be infringed only as a
482	+ consequence of further modification of the contributor version. For
483	+ purposes of this definition, "control" includes the right to grant
484	+ patent sublicenses in a manner consistent with the requirements of
485	+ this License.
486	+
487	+ Each contributor grants you a non-exclusive, worldwide, royalty-free
488	+ patent license under the contributor's essential patent claims, to
489	+ make, use, sell, offer for sale, import and otherwise run, modify and
490	+ propagate the contents of its contributor version.
491	+
492	+ In the following three paragraphs, a "patent license" is any express
493	+ agreement or commitment, however denominated, not to enforce a patent
494	+ (such as an express permission to practice a patent or covenant not to
495	+ sue for patent infringement). To "grant" such a patent license to a
496	+ party means to make such an agreement or commitment not to enforce a
497	+ patent against the party.
498	+
499	+ If you convey a covered work, knowingly relying on a patent license,
500	+ and the Corresponding Source of the work is not available for anyone
501	+ to copy, free of charge and under the terms of this License, through a
502	+ publicly available network server or other readily accessible means,
503	+ then you must either (1) cause the Corresponding Source to be so
504	+ available, or (2) arrange to deprive yourself of the benefit of the
505	+ patent license for this particular work, or (3) arrange, in a manner
506	+ consistent with the requirements of this License, to extend the patent
507	+ license to downstream recipients. "Knowingly relying" means you have
508	+ actual knowledge that, but for the patent license, your conveying the
509	+ covered work in a country, or your recipient's use of the covered work
510	+ in a country, would infringe one or more identifiable patents in that
511	+ country that you have reason to believe are valid.
512	+
513	+ If, pursuant to or in connection with a single transaction or
514	+ arrangement, you convey, or propagate by procuring conveyance of, a
515	+ covered work, and grant a patent license to some of the parties
516	+ receiving the covered work authorizing them to use, propagate, modify
517	+ or convey a specific copy of the covered work, then the patent license
518	+ you grant is automatically extended to all recipients of the covered
519	+ work and works based on it.
520	+
521	+ A patent license is "discriminatory" if it does not include within
522	+ the scope of its coverage, prohibits the exercise of, or is
523	+ conditioned on the non-exercise of one or more of the rights that are
524	+ specifically granted under this License. You may not convey a covered
525	+ work if you are a party to an arrangement with a third party that is
526	+ in the business of distributing software, under which you make payment
527	+ to the third party based on the extent of your activity of conveying
528	+ the work, and under which the third party grants, to any of the
529	+ parties who would receive the covered work from you, a discriminatory
530	+ patent license (a) in connection with copies of the covered work
531	+ conveyed by you (or copies made from those copies), or (b) primarily
532	+ for and in connection with specific products or compilations that
533	+ contain the covered work, unless you entered into that arrangement,
534	+ or that patent license was granted, prior to 28 March 2007.
535	+
536	+ Nothing in this License shall be construed as excluding or limiting
537	+ any implied license or other defenses to infringement that may
538	+ otherwise be available to you under applicable patent law.
539	+
540	+ 12. No Surrender of Others' Freedom.
541	+
542	+ If conditions are imposed on you (whether by court order, agreement or
543	+ otherwise) that contradict the conditions of this License, they do not
544	+ excuse you from the conditions of this License. If you cannot convey a
545	+ covered work so as to satisfy simultaneously your obligations under this
546	+ License and any other pertinent obligations, then as a consequence you may
547	+ not convey it at all. For example, if you agree to terms that obligate you
548	+ to collect a royalty for further conveying from those to whom you convey
549	+ the Program, the only way you could satisfy both those terms and this
550	+ License would be to refrain entirely from conveying the Program.
551	+
552	+ 13. Use with the GNU Affero General Public License.
553	+
554	+ Notwithstanding any other provision of this License, you have
555	+ permission to link or combine any covered work with a work licensed
556	+ under version 3 of the GNU Affero General Public License into a single
557	+ combined work, and to convey the resulting work. The terms of this
558	+ License will continue to apply to the part which is the covered work,
559	+ but the special requirements of the GNU Affero General Public License,
560	+ section 13, concerning interaction through a network will apply to the
561	+ combination as such.
562	+
563	+ 14. Revised Versions of this License.
564	+
565	+ The Free Software Foundation may publish revised and/or new versions of
566	+ the GNU General Public License from time to time. Such new versions will
567	+ be similar in spirit to the present version, but may differ in detail to
568	+ address new problems or concerns.
569	+
570	+ Each version is given a distinguishing version number. If the
571	+ Program specifies that a certain numbered version of the GNU General
572	+ Public License "or any later version" applies to it, you have the
573	+ option of following the terms and conditions either of that numbered
574	+ version or of any later version published by the Free Software
575	+ Foundation. If the Program does not specify a version number of the
576	+ GNU General Public License, you may choose any version ever published
577	+ by the Free Software Foundation.
578	+
579	+ If the Program specifies that a proxy can decide which future
580	+ versions of the GNU General Public License can be used, that proxy's
581	+ public statement of acceptance of a version permanently authorizes you
582	+ to choose that version for the Program.
583	+
584	+ Later license versions may give you additional or different
585	+ permissions. However, no additional obligations are imposed on any
586	+ author or copyright holder as a result of your choosing to follow a
587	+ later version.
588	+
589	+ 15. Disclaimer of Warranty.
590	+
591	+ THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
592	+ APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
593	+ HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
594	+ OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
595	+ THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
596	+ PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
597	+ IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
598	+ ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
599	+
600	+ 16. Limitation of Liability.
601	+
602	+ IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
603	+ WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
604	+ THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
605	+ GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
606	+ USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
607	+ DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
608	+ PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
609	+ EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
610	+ SUCH DAMAGES.
611	+
612	+ 17. Interpretation of Sections 15 and 16.
613	+
614	+ If the disclaimer of warranty and limitation of liability provided
615	+ above cannot be given local legal effect according to their terms,
616	+ reviewing courts shall apply local law that most closely approximates
617	+ an absolute waiver of all civil liability in connection with the
618	+ Program, unless a warranty or assumption of liability accompanies a
619	+ copy of the Program in return for a fee.
620	+
621	+ END OF TERMS AND CONDITIONS
622	+
623	+ How to Apply These Terms to Your New Programs
624	+
625	+ If you develop a new program, and you want it to be of the greatest
626	+ possible use to the public, the best way to achieve this is to make it
627	+ free software which everyone can redistribute and change under these terms.
628	+
629	+ To do so, attach the following notices to the program. It is safest
630	+ to attach them to the start of each source file to most effectively
631	+ state the exclusion of warranty; and each file should have at least
632	+ the "copyright" line and a pointer to where the full notice is found.
633	+
634	+ {one line to give the program's name and a brief idea of what it does.}
635	+ Copyright (C) {year} {name of author}
636	+
637	+ This program is free software: you can redistribute it and/or modify
638	+ it under the terms of the GNU General Public License as published by
639	+ the Free Software Foundation, either version 3 of the License, or
640	+ (at your option) any later version.
641	+
642	+ This program is distributed in the hope that it will be useful,
643	+ but WITHOUT ANY WARRANTY; without even the implied warranty of
644	+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
645	+ GNU General Public License for more details.
646	+
647	+ You should have received a copy of the GNU General Public License
648	+ along with this program. If not, see <http://www.gnu.org/licenses/>.
649	+
650	+ Also add information on how to contact you by electronic and paper mail.
651	+
652	+ If the program does terminal interaction, make it output a short
653	+ notice like this when it starts in an interactive mode:
654	+
655	+ Matomo Copyright (C) 2007-2018 Matomo.org
656	+
657	+ This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
658	+ This is free software, and you are welcome to redistribute it
659	+ under certain conditions; type `show c' for details.
660	+
661	+ The hypothetical commands `show w' and `show c' should show the appropriate
662	+ parts of the General Public License. Of course, your program's commands
663	+ might be different; for a GUI interface, you would use an "about box".
664	+
665	+ You should also get your employer (if you work as a programmer) or school,
666	+ if any, to sign a "copyright disclaimer" for the program, if necessary.
667	+ For more information on this, and how to apply and follow the GNU GPL, see
668	+ <http://www.gnu.org/licenses/>.
669	+
670	+ The GNU General Public License does not permit incorporating your program
671	+ into proprietary programs. If your program is a subroutine library, you
672	+ may consider it more useful to permit linking proprietary applications with
673	+ the library. If this is what you want to do, use the GNU Lesser General
674	+ Public License instead of this License. But first, please read
675	+ <http://www.gnu.org/philosophy/why-not-lgpl.html>.

app/PRIVACY.md ADDED Viewed

	@@ -0,0 +1,60 @@


1	+ # Privacy
2	+ This is a summary of all of the components within Matomo which may affect your privacy in some way. Please keep in mind
3	+ third party Themes, Plugins or Apps may introduce privacy concerns not listed here.
4	+
5	+ ## Privacy for users being tracked by Matomo
6	+ In this section we document how to protect the privacy of visitors who are tracked by your Matomo analytics service.
7	+
8	+ ### Anonymise visitor IP addresses
9	+ By default, Matomo stores the visitor IP address (IPv4 or IPv6 format) in the database for each new visitor.
10	+ If a visitor has a static IP address this means her browsing history can be easily identified across several days and
11	+ even across several websites tracked within the same Matomo server. You can anonymize IP addresses to ensure visitors cannot
12	+ be tracked this way: [How to anonymise IP addresses.](https://matomo.org/docs/privacy/#step-1-automatically-anonymize-visitor-ips)
13	+
14	+ ### Delete old visitors logs
15	+ By default, Matomo stores tracked data forever. To better respect the privacy of your users, it is recommended to regularly
16	+ purge old data. You can configure Matomo to automatically delete log data older than a specified number of months:
17	+ [How to delete old visitors log data.](https://matomo.org/docs/privacy/#step-2-delete-old-visitors-logs)
18	+
19	+ ### Include a tracking Opt-Out feature on your site
20	+ In your website, we recommended providing an easy way for your visitors to “opt-out” of being tracked by Matomo.
21	+ You can use the Opt-Out feature to display a link your website that sets a special browser cookie (`piwik_ignore`) when
22	+ clicked. Visitors that click that link will be ignored by Matomo in the future:
23	+ [How to include a tracking opt-out iframe.](https://matomo.org/docs/privacy/#step-3-include-a-web-analytics-opt-out-feature-on-your-site-using-an-iframe)
24	+
25	+ ### Respect DoNotTrack preference
26	+ Do Not Track is a browser-level technology and policy proposal that lets visitors opt out of tracking by websites they
27	+ do not visit. Visitors can enable this preference in their browser, and then it's up to Matomo to respect it. By default,
28	+ Matomo is configured to ignore visitors that have enabled it:
29	+ [How to check if your Matomo respects DoNotTrack.] (https://matomo.org/docs/privacy/#step-4-respect-donottrack-preference)
30	+
31	+ ### Disable tracking cookies
32	+ A cookie is a collection of information that a website stores on a visitor’s computer and accesses each time the visitor
33	+ returns. By default, Matomo uses cookies to aid in tracking visitor behavior. If someone gains access to a visitor's
34	+ computer, they could learn a few things about how the visitor visited your website. For many websites, this isn't a
35	+ problem, but for others where a strong level of privacy is required (like online banking), disabling tracking cookies may
36	+ be a good idea: [How to disable tracking cookies.](https://matomo.org/faq/general/faq_157/)
37	+
38	+ ### Keep your visitors details private
39	+ Any user that has at least `view` access (the default access level) to Matomo can view detailed information for all users
40	+ tracked in Matomo (such as their IP addresses, visitor IDs, details of all past visits and actions, etc.) through features
41	+ provided by the `Live` plugin (such as the Visitor Log and Visitor Profile). As the Matomo administrator, you may decide
42	+ that not all of your users need access to this data. You can deactivate the `Live` plugin to prevent users from viewing
43	+ visitor details in the Administration > Plugins page.
44	+
45	+ ## Privacy for Matomo admins and website owners
46	+ In this section we document how a Matomo administrator can better protect their own privacy.
47	+
48	+ ### Keep your Matomo server URL private
49	+ By default, the Matomo Javascript code on all tracked websites contains the Matomo server URL. In some cases you might
50	+ want to hide this Matomo URL completely while still tracking all websites in your Matomo instance. To hide your Matomo
51	+ server's URL, you can modify the Javascript Tracking code and point it to a proxy piwik.php script instead of your actual
52	+ Matomo server: [How to keep Matomo server URL private.](https://matomo.org/faq/how-to/faq_132/)
53	+
54	+ ### Automatic update check
55	+ From time to time, Matomo uses `api.matomo.org` to check if the current version of Matomo is the latest version of Matomo.
56	+ If an update is available, a notification is displayed allowing you to upgrade Matomo. To disable the update check,
57	+ and stop your instance from sending HTTP requests to `api.matomo.org`, deactivate the "Automatic update" feature by
58	+ setting `enable_auto_update = 0` in your configuration file `config/config.ini.php`.
59	+
60	+ Learn more about [Privacy in Matomo](https://matomo.org/privacy/).

app/README.md ADDED Viewed

	@@ -0,0 +1,119 @@


1	+ # Matomo (formerly Piwik) - matomo.org
2	+
3	+ [![Latest Stable Version](https://poser.pugx.org/piwik/piwik/v/stable)](https://matomo.org/download/)
4	+ [![Latest Unstable Version](https://poser.pugx.org/piwik/piwik/v/unstable)](https://packagist.org/packages/piwik/piwik)
5	+ [![License](https://poser.pugx.org/piwik/piwik/license)](https://matomo.org/free-software/)
6	+
7	+ ## Code Status
8	+
9	+ [![Build Status](https://travis-ci.org/matomo-org/matomo.svg?branch=master)](https://travis-ci.org/matomo-org/matomo/branches)
10	+ [![Percentage of issues still open](http://isitmaintained.com/badge/open/matomo-org/matomo.svg)](http://isitmaintained.com/project/matomo-org/matomo "Percentage of issues still open")
11	+
12	+ ## Description
13	+
14	+ Matomo is the leading Free/Libre open analytics platform.
15	+
16	+ Matomo is a full-featured PHP MySQL software program that you download and install on your own webserver.
17	+ At the end of the five-minute installation process, you will be given a JavaScript code.
18	+ Simply copy and paste this tag on websites you wish to track and access your analytics reports in real-time.
19	+
20	+ Matomo aims to be a Free software alternative to Google Analytics and is already used on more than 1,400,000 websites. Privacy is built-in!
21	+
22	+ ## Mission Statement
23	+
24	+ > « To create, as a community, the leading international open source digital analytics platform, that gives every user full control of their data. »
25	+
26	+ Or in short:
27	+ > « Liberate Web Analytics »
28	+
29	+ ## License
30	+
31	+ Matomo is released under the GPL v3 (or later) license, see [LICENSE](LICENSE)
32	+
33	+ ## Requirements
34	+
35	+ * PHP 5.5.9 or greater
36	+ * MySQL version 5.5 or greater, or MariaDB
37	+ * PHP extension pdo and pdo_mysql, or the MySQLi extension.
38	+ * Matomo is OS / server independent
39	+
40	+ See https://matomo.org/docs/requirements/
41	+
42	+ ## Install Matomo
43	+
44	+ * [Download Matomo](https://matomo.org/download/)
45	+ * Upload matomo to your webserver
46	+ * Point your browser to the directory
47	+ * Follow the steps
48	+ * Add the given javascript code to your pages
49	+ * (You may also generate fake data to experiment, by enabling the plugin VisitorGenerator)
50	+
51	+ See https://matomo.org/docs/installation/
52	+
53	+ (When using Matomo for development you need to [install Matomo from the Git repository.](https://matomo.org/faq/how-to-install/faq_18271/))
54	+
55	+ ## Free trial
56	+
57	+ If you do not have a server or don't want to host yourself you can use our Matomo Cloud partner service (21 day free trial): https://matomo.org/start-free-analytics-trial/
58	+
59	+ ## Online Demo
60	+
61	+ Check out the online demo for Matomo at [demo.matomo.org](https://demo.matomo.org/)
62	+
63	+ ## Changelog
64	+
65	+ For the list of all tickets closed in the current and past releases, see [matomo.org/changelog/](https://matomo.org/changelog/). For the list of technical changes in the Matomo platform, see [developer.matomo.org/changelog](https://developer.matomo.org/changelog).
66	+
67	+ ## Get involved!
68	+
69	+ We believe in liberating Web Analytics, providing a free platform for simple and advanced analytics. Matomo was built by dozens of people like you,
70	+ and we need your help to make Matomo better… Why not participate in a useful project today? [Learn how you can contribute to Matomo.](https://matomo.org/get-involved)
71	+
72	+ ## Quality Assurance
73	+
74	+ The Matomo project uses an ever-expanding comprehensive set of thousands of unit tests and hundreds of automated integration tests, system tests, JavaScript tests, and screenshot UI tests, running on a continuous integration server as part of its software quality assurance. [Learn more](https://developer.matomo.org/guides/tests)
75	+
76	+ We use [BrowserStack.com](https://www.browserstack.com/) testing tool to help check the Matomo user interface is compatible with many browsers.
77	+
78	+ ## Security
79	+
80	+ Security is a top priority at Matomo. As potential issues are discovered, we validate, patch and release fixes as quickly as we can. We have a security bug bounty program in place that rewards researchers for finding security issues and disclosing them to us.
81	+
82	+ [Learn more](https://matomo.org/security/) or check out our [HackerOne program](https://hackerone.com/matomo).
83	+
84	+ ## Support for Matomo
85	+
86	+ For Free support, post a message in our community forums: [forum.matomo.org](https://forum.matomo.org/)
87	+
88	+ For Professional paid support, purchase a Matomo On-Premises Support Plan: [matomo.org/support-plans](https://matomo.org/support-plans/)
89	+
90	+ ## Contact
91	+
92	+ Website: [matomo.org](https://matomo.org)
93	+
94	+ About us: [matomo.org/team/](https://matomo.org/team/)
95	+
96	+ Contact us: [matomo.org/contact/](https://matomo.org/contact/)
97	+
98	+
99	+ ## More information
100	+
101	+ What makes Matomo unique from the competition:
102	+
103	+ * You own your web analytics data: since Matomo is installed on your server, the data is stored in your own database and you can get all the statistics using the powerful Matomo Analytics API.
104	+
105	+ * Matomo is a Free Software which can easily be configured to respect your visitors' privacy.
106	+
107	+ * Modern, easy to use User Interface: you can fully customize your dashboard, drag and drop widgets and more.
108	+
109	+ * Matomo features are built inside plugins: you can add new features and remove the ones you don’t need.
110	+ You can build your own web analytics plugins or hire a consultant to have your custom feature built-in Matomo.
111	+
112	+ * A vibrant international Open community of more than 200,000 active users (tracking even more websites!)
113	+
114	+ * Advanced Web Analytics capabilities such as E-commerce Tracking, Goal tracking, Campaign tracking,
115	+ Custom Variables, Email Reports, Custom Segment Editor, Geo Location, Real-time visits and maps, [and a lot more!](https://matomo.org/feature-overview/)
116	+
117	+ Documentation and more info on https://matomo.org
118	+
119	+ We are together creating the best open analytics platform in the world!

app/SECURITY.md ADDED Viewed

	@@ -0,0 +1,23 @@


1	+ # Reporting Security Issues
2	+
3	+ ## Security Bug Bounty Program
4	+
5	+ The Matomo Security Bug Bounty Program is designed to encourage security research in Matomo software and to reward those who help us create the safest web analytics platform. The bounty for valid critical security bugs is a $777 (US) cash reward. The bounty for non-critical bugs is $333 (US), paid via Paypal.
6	+
7	+
8	+ ## Responsible disclosure by email
9	+
10	+
11	+ We encourage you to responsibly report issues via our [Matomo Bug Bounty Program on HackerOne](https://hackerone.com/matomo) or you can also
12	+ [email us at security@matomo.org](mailto:security@matomo.org?subject=Reporting%20Vulnerability%20in%20Matomo).
13	+
14	+ If you have found a security issue in Matomo please read [our security notes](https://matomo.org/security/) regarding responsible disclosures.
15	+
16	+
17	+ ## Improve your Matomo Server Security
18	+
19	+ [Secure Matomo server](https://matomo.org/docs/security/): follow these steps to keep your Matomo data safe.
20	+
21	+ ## Security announcements
22	+
23	+ Please subscribe to [the Changelog](https://matomo.org/changelog/) ([rss feed](https://matomo.org/changelog/feed/)) to be notified of new releases (including security releases).

app/bootstrap.php ADDED Viewed

	@@ -0,0 +1,202 @@


1	+ <?php
2	+
3	+ $GLOBALS['CONFIG_INI_PATH_RESOLVER'] = function () {
4	+ if ( defined( 'ABSPATH' )
5	+ && defined( 'MATOMO_CONFIG_PATH' ) ) {
6	+ $paths = new \WpMatomo\Paths();
7	+
8	+ return $paths->get_config_ini_path();
9	+ }
10	+ };
11	+
12	+ $matomo_is_archive_request = !empty($_SERVER['argv'])
13	+ && is_array($_SERVER['argv'])
14	+ && in_array('climulti:request', $_SERVER['argv'], true);
15	+
16	+ if ( ! defined( 'PIWIK_ENABLE_ERROR_HANDLER' ) ) {
17	+ // we prefer using WP error handler... unless we are archiving where we want to prevent any warnings being printed
18	+ // as otherwise the archiving would be marked as failed because the cli archive output would contain a warning and
19	+ // the output would not be possible to do an unserialize anymore
20	+ if (!$matomo_is_archive_request) {
21	+ define( 'PIWIK_ENABLE_ERROR_HANDLER', false );
22	+ }
23	+ }
24	+
25	+ $matomo_was_wp_loaded_directly = ! defined( 'ABSPATH' );
26	+
27	+
28	+ function matomo_log_message_no_display($message)
29	+ {
30	+ $message = 'Matomo ' . $message;
31	+
32	+ if ( defined( 'WP_DEBUG' ) && WP_DEBUG === true ) {
33	+ if (function_exists('ini_set') && function_exists('ini_get')) {
34	+ $value_orig = @ini_get('display_errors');
35	+ $value = @ini_set('display_errors', 'Off');
36	+ if (false !== $value) {
37	+ error_log( $message );
38	+ }
39	+ @ini_set('display_errors', $value_orig);
40	+ }
41	+ }
42	+
43	+ if (function_exists('update_option')
44	+ && class_exists('\WpMatomo\Logger')) {
45	+ // only if WordPress was bootstrapped by now... otherwise it will fail
46	+ try {
47	+ $logger = new \WpMatomo\Logger();
48	+ $logger->log_exception('archive_boot', new Exception($message));
49	+ } catch (Exception $e) {
50	+
51	+ }
52	+ }
53	+ }
54	+
55	+ if ( $matomo_was_wp_loaded_directly ) {
56	+ // prevent from loading twice
57	+ $matomo_wpload_base = '../../../../wp-load.php';
58	+ $matomo_wpload_full = dirname( __FILE__ ) . '/' . $matomo_wpload_base;
59	+
60	+ if ($matomo_is_archive_request) {
61	+ ob_start();
62	+ // the matomo error handler will be only loaded after WordPress has been loaded... here we want to prevent
63	+ // any warning/notice from being shown while bootstrapping WordPress or otherwise the unserialize of the response
64	+ // later in climulti will fail
65	+ set_error_handler(function ($errno, $errstr, $errfile, $errline) {
66	+ // if the error has been suppressed by the @ we don't handle the error
67	+ if (error_reporting() == 0) {
68	+ return;
69	+ }
70	+
71	+ if (in_array($errno, array(E_ERROR, E_PARSE, E_CORE_ERROR, E_CORE_WARNING, E_COMPILE_ERROR, E_COMPILE_WARNING, E_USER_ERROR))) {
72	+ return false; //force standard behaviour
73	+ }
74	+
75	+ matomo_log_message_no_display( sprintf('error: %s: %s in %s:%s', $errno, $errstr, $errfile, $errline ) );
76	+ });
77	+ }
78	+
79	+ if (!empty($_ENV['MATOMO_WP_ROOT_PATH']) && file_exists( rtrim($_ENV['MATOMO_WP_ROOT_PATH'], '/') . '/wp-load.php')) {
80	+ require_once rtrim($_ENV['MATOMO_WP_ROOT_PATH'], '/') . '/wp-load.php';
81	+ } elseif ( file_exists($matomo_wpload_full ) ) {
82	+ require_once $matomo_wpload_full;
83	+ } elseif (realpath( $matomo_wpload_full ) && file_exists(realpath( $matomo_wpload_full ))) {
84	+ require_once realpath( $matomo_wpload_full );
85	+ } elseif (!empty($_SERVER['SCRIPT_FILENAME']) && file_exists($_SERVER['SCRIPT_FILENAME'])) {
86	+ // seems symlinked... eg the wp-content dir or wp-content/plugins dir is symlinked from some very much other place...
87	+ $matomo_wpload_full = dirname($_SERVER['SCRIPT_FILENAME']) . '/' . $matomo_wpload_base;
88	+ if ( file_exists($matomo_wpload_full ) ) {
89	+ require_once $matomo_wpload_full;
90	+ } elseif (realpath( $matomo_wpload_full ) && file_exists(realpath( $matomo_wpload_full ))) {
91	+ require_once realpath( $matomo_wpload_full );
92	+ } elseif (file_exists(dirname(dirname(dirname(dirname(dirname( $_SERVER['SCRIPT_FILENAME'] ))))) . '/wp-load.php')) {
93	+ require_once dirname(dirname(dirname(dirname(dirname( $_SERVER['SCRIPT_FILENAME'] ))))) . '/wp-load.php';
94	+ }
95	+ }
96	+
97	+ if ($matomo_is_archive_request) {
98	+ restore_error_handler();
99	+ if (ob_get_level()) {
100	+ $matomo_ob_end_clean_msg = @ob_get_clean();
101	+ if (!empty($matomo_ob_end_clean_msg)) {
102	+ matomo_log_message_no_display( $matomo_ob_end_clean_msg );
103	+ }
104	+ }
105	+ }
106	+ }
107	+
108	+ if ( ! defined( 'ABSPATH' ) ) {
109	+ echo 'Could not find wp-load. If your server uses symlinks or a custom content directory, Matomo may not work for you as we cannot detect the paths correctly. For more information see https://matomo.org/faq/wordpress/what-are-the-requirements-for-matomo-for-wordpress/';
110	+ exit; // if accessed directly
111	+ }
112	+
113	+ if ( !is_plugin_active('matomo/matomo.php')
114	+ && (!defined( 'MATOMO_PHPUNIT_TEST' ) \|\| !MATOMO_PHPUNIT_TEST) ) { // during tests the plugin may temporarily not be active
115	+ exit;
116	+ }
117	+
118	+ if ($matomo_was_wp_loaded_directly) {
119	+ // see https://github.com/matomo-org/wp-matomo/issues/190
120	+ // wp-external-links plugin would register an ob_start(function () {...}) and manipulate any of our API output
121	+ // and in some cases the output would get completely lost causing blank pages.
122	+ add_filter('wpel_apply_settings', '__return_false', 99999);
123	+
124	+ // do not strip slashes if we bootstrap matomo within a regular wordpress request
125	+ if (!empty($_GET)) {
126	+ $_GET = stripslashes_deep( $_GET );
127	+ }
128	+ if (!empty($_POST)) {
129	+ $_POST = stripslashes_deep( $_POST );
130	+ }
131	+ if (!empty($_COOKIE)) {
132	+ $_COOKIE = stripslashes_deep( $_COOKIE );
133	+ }
134	+ if (!empty($_SERVER)) {
135	+ $_SERVER = stripslashes_deep( $_SERVER );
136	+ }
137	+ if (!empty($_REQUEST)) {
138	+ $_REQUEST = stripslashes_deep( $_REQUEST );
139	+ }
140	+ }
141	+
142	+
143	+ if ( matomo_is_app_request() ) {
144	+ // pretend we are in the admin... potentially avoiding caching etc
145	+ $GLOBALS['hook_suffix'] = '';
146	+ include_once ABSPATH . '/wp-admin/includes/class-wp-screen.php';
147	+ $GLOBALS['current_screen'] = WP_Screen::get();
148	+
149	+ // we disable jsonp
150	+ unset($_GET['jsoncallback']);
151	+ unset($_GET['callback']);
152	+ unset($_POST['jsoncallback']);
153	+ unset($_POST['callback']);
154	+ }
155	+
156	+ if ( ! defined( 'PIWIK_USER_PATH' ) ) {
157	+ define( 'PIWIK_USER_PATH', dirname( MATOMO_ANALYTICS_FILE ) );
158	+ }
159	+
160	+ if (function_exists('wp_raise_memory_limit') && function_exists('wp_convert_hr_to_bytes')) {
161	+ $current_limit = ini_get( 'memory_limit' );
162	+ $current_limit_int = wp_convert_hr_to_bytes( $current_limit );
163	+ $memory128MbInt = 134217728;
164	+ if ($current_limit_int && $current_limit_int > 0 && $current_limit_int < $memory128MbInt) {
165	+ // we try increase memory if memory is less than 128mb
166	+ wp_raise_memory_limit('admin');
167	+ }
168	+ }
169	+
170	+ $GLOBALS['MATOMO_MODIFY_CONFIG_SETTINGS'] = function ($settings) {
171	+ $plugins = $settings['Plugins'];
172	+ if (is_array($settings['Plugins'])) {
173	+ $pluginsToRemove = array('Marketplace', 'MultiSites', 'TwoFactorAuth', 'Widgetize', 'Monolog', 'Feedback', 'ExamplePlugin', 'ExampleAPI', 'ProfessionalServices', 'MobileAppMeasurable');
174	+ foreach ($pluginsToRemove as $pluginToRemove) {
175	+ // Marketplace => this is instead done in wordpress
176	+ // MultiSites => doesn't really make sense since we have only one website per installation
177	+ // TwoFactorAuth => not needed as login is being handled by WordPress
178	+ // widgetize for now we don't want to allow widgetizing as it is based on the token_auth authentication
179	+ // Monolog => we use our own logger
180	+ // ProfessionalServices => we advertise in the WP plugin itself instead
181	+ // feedback => we want to hide things like Need help in the admin etc
182	+ // MobileAppMeasurable => for WP mobile apps are not a thing
183	+ // custom variables we don't want to enable as we will deprecate them in Matomo 4 anyway => used to be disabled but we need to make sure the columns get installed otherwise matomo has issues... need to wait to matomo 4 to remove it
184	+ $pos = array_search($pluginToRemove, $plugins['Plugins']);
185	+ if ($pos !== false) {
186	+ array_splice($plugins['Plugins'], $pos, 1);
187	+ }
188	+ }
189	+ if (matomo_has_tag_manager()) {
190	+ $plugins['Plugins'][] = 'TagManager';
191	+ }
192	+ }
193	+ if (!empty($GLOBALS['MATOMO_PLUGINS_ENABLED'])) {
194	+ foreach ($GLOBALS['MATOMO_PLUGINS_ENABLED'] as $plugin) {
195	+ if (!in_array($plugin, $plugins['Plugins'])) {
196	+ $plugins['Plugins'][] = $plugin;
197	+ }
198	+ }
199	+ }
200	+ $settings['Plugins'] = $plugins;
201	+ return $settings;
202	+ };

app/config/environment/dev.php ADDED Viewed

	@@ -0,0 +1,12 @@


1	+ <?php
2	+
3	+ return array(
4	+
5	+ 'Piwik\Cache\Backend' => DI\object('Piwik\Cache\Backend\ArrayCache'),
6	+
7	+ 'Piwik\Translation\Loader\LoaderInterface' => DI\object('Piwik\Translation\Loader\LoaderCache')
8	+ ->constructor(DI\get('Piwik\Translation\Loader\DevelopmentLoader')),
9	+ 'Piwik\Translation\Loader\DevelopmentLoader' => DI\object()
10	+ ->constructor(DI\get('Piwik\Translation\Loader\JsonFileLoader')),
11	+
12	+ );

app/config/global.ini.php ADDED Viewed

	@@ -0,0 +1,1033 @@


1	+ ; <?php exit; ?> DO NOT REMOVE THIS LINE
2	+ ; If you want to change some of these default values, the best practise is to override
3	+ ; them in your configuration file in config/config.ini.php. If you directly edit this file,
4	+ ; you will lose your changes when you upgrade Matomo.
5	+ ; For example if you want to override action_title_category_delimiter,
6	+ ; edit config/config.ini.php and add the following:
7	+ ; [General]
8	+ ; action_title_category_delimiter = "-"
9	+
10	+ ;--------
11	+ ; WARNING - YOU SHOULD NOT EDIT THIS FILE DIRECTLY - Edit config.ini.php instead.
12	+ ;--------
13	+
14	+ [database]
15	+ host =
16	+ username =
17	+ password =
18	+ dbname =
19	+ tables_prefix =
20	+ port = 3306
21	+ adapter = PDO\MYSQL
22	+ type = InnoDB
23	+ schema = Mysql
24	+
25	+ ; Database SSL Options START
26	+ ; Turn on or off SSL connection to database, possible values for enable_ssl: 1 or 0
27	+ enable_ssl = 0
28	+ ; Direct path to server CA file, CA bundle supported (required for ssl connection)
29	+ ssl_ca =
30	+ ; Direct path to client cert file (optional)
31	+ ssl_cert =
32	+ ; Direct path to client key file (optional)
33	+ ssl_key =
34	+ ; Direct path to CA cert files directory (optional)
35	+ ssl_ca_path =
36	+ ; List of one or more ciphers for SSL encryption, in OpenSSL format (optional)
37	+ ssl_cipher =
38	+ ; Whether to skip verification of self signed certificates (optional, only supported
39	+ ; w/ specific PHP versions, and is mostly for testing purposes)
40	+ ssl_no_verify =
41	+ ; Database SSL Options END
42	+
43	+ ; if charset is set to utf8, Matomo will ensure that it is storing its data using UTF8 charset.
44	+ ; it will add a sql query SET at each page view.
45	+ ; Matomo should work correctly without this setting but we recommend to have a charset set.
46	+ charset = utf8
47	+
48	+ ; If configured, the following queries will be executed on the reader instead of the writer.
49	+ ; * archiving queries that hit a log table
50	+ ; * live queries that hit a log table
51	+ ; You only want to enable a reader if you can ensure there is minimal replication lag / delay on the reader.
52	+ ; Otherwise you might get corrupt data in the reports.
53	+ [database_reader]
54	+ host =
55	+ username =
56	+ password =
57	+ dbname =
58	+ port = 3306
59	+
60	+ [database_tests]
61	+ host = localhost
62	+ username = "@USERNAME@"
63	+ password =
64	+ dbname = matomo_tests
65	+ tables_prefix = matomotests_
66	+ port = 3306
67	+ adapter = PDO\MYSQL
68	+ type = InnoDB
69	+ schema = Mysql
70	+ charset = utf8
71	+ enable_ssl = 0
72	+ ssl_ca =
73	+ ssl_cert =
74	+ ssl_key =
75	+ ssl_ca_path =
76	+ ssl_cipher =
77	+ ssl_no_verify = 1
78	+
79	+ [tests]
80	+ ; needed in order to run tests.
81	+ ; if Matomo is available at http://localhost/dev/matomo/ replace @REQUEST_URI@ with /dev/matomo/
82	+ ; note: the REQUEST_URI should not contain "plugins" or "tests" in the PATH
83	+ http_host = localhost
84	+ remote_addr = "127.0.0.1"
85	+ request_uri = "@REQUEST_URI@"
86	+ port =
87	+ enable_logging = 0
88	+
89	+ ; access key and secret as listed in AWS -> IAM -> Users
90	+ aws_accesskey = ""
91	+ aws_secret = ""
92	+ ; key pair name as listed in AWS -> EC2 -> Key Pairs. Key name should be different per user.
93	+ aws_keyname = ""
94	+ ; PEM file can be downloaded after creating a new key pair in AWS -> EC2 -> Key Pairs
95	+ aws_pem_file = "<path to pem file>"
96	+ aws_securitygroups[] = "default"
97	+ aws_region = "us-east-1"
98	+ aws_ami = "ami-ac24bac4"
99	+ aws_instance_type = "c3.large"
100	+
101	+ [log]
102	+ ; possible values for log: screen, database, file
103	+ log_writers[] = screen
104	+
105	+ ; log level, everything logged w/ this level or one of greater severity
106	+ ; will be logged. everything else will be ignored. possible values are:
107	+ ; ERROR, WARN, INFO, DEBUG
108	+ ; this setting will apply to every log writer, if there is no specific log level defined for a writer.
109	+ log_level = WARN
110	+
111	+ ; you can also set specific log levels for different writers, by appending the writer name to log_level_, like so:
112	+ ; this allows you to log more information to one backend vs another.
113	+ ; log_level_screen =
114	+ ; log_level_file =
115	+
116	+ ; if configured to log in a file, log entries will be made to this file
117	+ logger_file_path = tmp/logs/matomo.log
118	+
119	+ [Cache]
120	+ ; available backends are 'file', 'array', 'null', 'redis', 'chained'
121	+ ; 'array' will cache data only during one request
122	+ ; 'null' will not cache anything at all
123	+ ; 'file' will cache on the filesystem
124	+ ; 'redis' will cache on a Redis server, use this if you are running Matomo with multiple servers. Further configuration in [RedisCache] is needed
125	+ ; 'chained' will chain multiple cache backends. Further configuration in [ChainedCache] is needed
126	+ backend = chained
127	+
128	+ [ChainedCache]
129	+ ; The chained cache will always try to read from the fastest backend first (the first listed one) to avoid requesting
130	+ ; the same cache entry from the slowest backend multiple times in one request.
131	+ backends[] = array
132	+ backends[] = file
133	+
134	+ [RedisCache]
135	+ ; Redis server configuration.
136	+ host = "127.0.0.1"
137	+ port = 6379
138	+ ; instead of host and port a unix socket path can be configured
139	+ unix_socket = ""
140	+ timeout = 0.0
141	+ password = ""
142	+ database = 14
143	+ ; In case you are using queued tracking: Make sure to configure a different database! Otherwise queued requests might
144	+ ; be flushed
145	+
146	+ [Debug]
147	+ ; if set to 1, the archiving process will always be triggered, even if the archive has already been computed
148	+ ; this is useful when making changes to the archiving code so we can force the archiving process
149	+ always_archive_data_period = 0;
150	+ always_archive_data_day = 0;
151	+ ; Force archiving Custom date range (without re-archiving sub-periods used to process this date range)
152	+ always_archive_data_range = 0;
153	+
154	+ ; if set to 1, all the SQL queries will be recorded by the profiler
155	+ ; and a profiling summary will be printed at the end of the request
156	+ ; NOTE: you must also set [log] log_writers[] = "screen" to enable the profiler to print on screen
157	+ enable_sql_profiler = 0
158	+
159	+ ; If set to 1, all requests to matomo.php will be forced to be 'new visitors'
160	+ tracker_always_new_visitor = 0
161	+
162	+ ; if set to 1, all SQL queries will be logged using the DEBUG log level
163	+ log_sql_queries = 0
164	+
165	+ ; if set to 1, core:archive profiling information will be recorded in a log file. the log file is determined by the
166	+ ; archive_profiling_log option.
167	+ archiving_profile = 0
168	+
169	+ ; if set to an absolute path, core:archive profiling information will be logged to specified file
170	+ archive_profiling_log =
171	+
172	+ [DebugTests]
173	+ ; When set to 1, standalone plugins (those with their own git repositories)
174	+ ; will be loaded when executing tests.
175	+ enable_load_standalone_plugins_during_tests = 0
176	+
177	+ [Development]
178	+ ; Enables the development mode where we avoid most caching to make sure code changes will be directly applied as
179	+ ; some caches are only invalidated after an update otherwise. When enabled it'll also performs some validation checks.
180	+ ; For instance if you register a method in a widget we will verify whether the method actually exists and is public.
181	+ ; If not, we will show you a helpful warning to make it easy to find simple typos etc.
182	+ enabled = 0
183	+
184	+ ; if set to 1, javascript files will be included individually and neither merged nor minified.
185	+ ; this option must be set to 1 when adding, removing or modifying javascript files
186	+ ; Note that for quick debugging, instead of using below setting, you can add `&disable_merged_assets=1` to the Matomo URL
187	+ disable_merged_assets = 0
188	+
189	+ [General]
190	+ ; the following settings control whether Unique Visitors `nb_uniq_visitors` and Unique users `nb_users` will be processed for different period types.
191	+ ; year and range periods are disabled by default, to ensure optimal performance for high traffic Matomo instances
192	+ ; if you set it to 1 and want the Unique Visitors to be re-processed for reports in the past, drop all matomo_archive_* tables
193	+ ; it is recommended to always enable Unique Visitors and Unique Users processing for 'day' periods
194	+ enable_processing_unique_visitors_day = 1
195	+ enable_processing_unique_visitors_week = 1
196	+ enable_processing_unique_visitors_month = 1
197	+ enable_processing_unique_visitors_year = 0
198	+ enable_processing_unique_visitors_range = 0
199	+
200	+ ; controls whether Unique Visitors will be processed for groups of websites. these metrics describe the number
201	+ ; of unique visitors across the entire set of websites, so if a visitor visited two websites in the group, she
202	+ ; would still only be counted as one. only relevant when using plugins that group sites together
203	+ enable_processing_unique_visitors_multiple_sites = 0
204	+
205	+ ; The list of periods that are available in the Matomo calendar
206	+ ; Example use case: custom date range requests are processed in real time,
207	+ ; so they may take a few minutes on very high traffic website: you may remove "range" below to disable this period
208	+ enabled_periods_UI = "day,week,month,year,range"
209	+ enabled_periods_API = "day,week,month,year,range"
210	+
211	+ ; whether to enable segment archiving cache
212	+ ; Note: if you use any plugins, this need to be compliant with Matomo and
213	+ ; * depending on the segment you create you may need a newer MySQL version (eg 5.7 or newer)
214	+ ; * use a reader database for archiving in case you have configured a database reader
215	+ enable_segments_cache = 1
216	+
217	+ ; whether to enable subquery cache for Custom Segment archiving queries
218	+ enable_segments_subquery_cache = 0
219	+ ; Any segment subquery that matches more than segments_subquery_cache_limit IDs will not be cached,
220	+ ; and the original subquery executed instead.
221	+ segments_subquery_cache_limit = 100000
222	+ ; TTL: Time to live for cache files, in seconds. Default to 60 minutes
223	+ segments_subquery_cache_ttl = 3600
224	+
225	+ ; when set to 1, all requests to Matomo will return a maintenance message without connecting to the DB
226	+ ; this is useful when upgrading using the shell command, to prevent other users from accessing the UI while Upgrade is in progress
227	+ maintenance_mode = 0
228	+
229	+ ; Defines the release channel that shall be used. Currently available values are:
230	+ ; "latest_stable", "latest_beta", "latest_3x_stable", "latest_3x_beta"
231	+ release_channel = "latest_stable"
232	+
233	+ ; character used to automatically create categories in the Actions > Pages, Outlinks and Downloads reports
234	+ ; for example a URL like "example.com/blog/development/first-post" will create
235	+ ; the page first-post in the subcategory development which belongs to the blog category
236	+ action_url_category_delimiter = /
237	+
238	+ ; similar to above, but this delimiter is only used for page titles in the Actions > Page titles report
239	+ action_title_category_delimiter = ""
240	+
241	+ ; the maximum url category depth to track. if this is set to 2, then a url such as
242	+ ; "example.com/blog/development/first-post" would be treated as "example.com/blog/development".
243	+ ; this setting is used mainly to limit the amount of data that is stored by Matomo.
244	+ action_category_level_limit = 10
245	+
246	+ ; minimum number of websites to run autocompleter
247	+ autocomplete_min_sites = 5
248	+
249	+ ; maximum number of websites showed in search results in autocompleter
250	+ site_selector_max_sites = 15
251	+
252	+ ; if set to 1, shows sparklines (evolution graph) in 'All Websites' report (MultiSites plugin)
253	+ show_multisites_sparklines = 1
254	+
255	+ ; number of websites to display per page in the All Websites dashboard
256	+ all_websites_website_per_page = 50
257	+
258	+ ; if set to 0, the anonymous user will not be able to use the 'segments' parameter in the API request
259	+ ; this is useful to prevent full DB access to the anonymous user, or to limit performance usage
260	+ anonymous_user_enable_use_segments_API = 1
261	+
262	+ ; if browser trigger archiving is disabled, API requests with a &segment= parameter will still trigger archiving.
263	+ ; You can force the browser archiving to be disabled in most cases by setting this setting to 1
264	+ ; The only time that the browser will still trigger archiving is when requesting a custom date range that is not pre-processed yet
265	+ browser_archiving_disabled_enforce = 0
266	+
267	+ ; Add custom currencies to Sites Manager.
268	+ currencies[BTC] = Bitcoin
269	+
270	+ ; By default, users can create Segments which are to be processed in Real-time.
271	+ ; Setting this to 0 will force all newly created Custom Segments to be "Pre-processed (faster, requires archive.php cron)"
272	+ ; This can be useful if you want to prevent users from adding much load on the server.
273	+ ; Notes:
274	+ ; * any existing Segment set to "processed in Real time", will still be set to Real-time.
275	+ ; this will only affect custom segments added or modified after this setting is changed.
276	+ ; * when set to 0 then any user with at least 'view' access will be able to create pre-processed segments.
277	+ enable_create_realtime_segments = 1
278	+
279	+ ; Whether to enable the "Suggest values for segment" in the Segment Editor panel.
280	+ ; Set this to 0 in case your Matomo database is very big, and suggested values may not appear in time
281	+ enable_segment_suggested_values = 1
282	+
283	+ ; By default, any user with a "view" access for a website can create segment assigned to this website.
284	+ ; Set this to "admin" or "superuser" to require that users should have at least this access to create new segments.
285	+ ; Note: anonymous user (even if it has view access) is not allowed to create or edit segment.
286	+ ; Possible values are "view", "admin", "superuser"
287	+ adding_segment_requires_access = "view"
288	+
289	+ ; Whether it is allowed for users to add segments that affect all websites or not. If there are many websites
290	+ ; this admin option can be used to prevent users from performing an action that will have a major impact
291	+ ; on Matomo performance.
292	+ allow_adding_segments_for_all_websites = 1
293	+
294	+ ; When archiving segments for the first time, this determines the oldest date that will be archived.
295	+ ; This option can be used to avoid archiving (for isntance) the lastN years for every new segment.
296	+ ; Valid option values include: "beginning_of_time" (start date of archiving will not be changed)
297	+ ; "segment_last_edit_time" (start date of archiving will be the earliest last edit date found,
298	+ ; if none is found, the created date is used)
299	+ ; "segment_creation_time" (start date of archiving will be the creation date of the segment)
300	+ ; lastN where N is an integer (eg "last10" to archive for 10 days before the segment creation date)
301	+ process_new_segments_from = "beginning_of_time"
302	+
303	+ ; this action name is used when the URL ends with a slash /
304	+ ; it is useful to have an actual string to write in the UI
305	+ action_default_name = index
306	+
307	+ ; default language to use in Matomo
308	+ default_language = en
309	+
310	+ ; default number of elements in the datatable
311	+ datatable_default_limit = 10
312	+
313	+ ; Each datatable report has a Row Limit selector at the bottom right.
314	+ ; By default you can select from 5 to 500 rows. You may customise the values below
315	+ ; -1 will be displayed as 'all' and it will export all rows (filter_limit=-1)
316	+ datatable_row_limits = "5,10,25,50,100,250,500,-1"
317	+
318	+ ; default number of rows returned in API responses
319	+ ; this value is overwritten by the '# Rows to display' selector.
320	+ ; if set to -1, a click on 'Export as' will export all rows independently of the current '# Rows to display'.
321	+ API_datatable_default_limit = 100
322	+
323	+ ; When period=range, below the datatables, when user clicks on "export", the data will be aggregate of the range.
324	+ ; Here you can specify the comma separated list of formats for which the data will be exported aggregated by day
325	+ ; (ie. there will be a new "date" column). For example set to: "rss,tsv,csv"
326	+ datatable_export_range_as_day = "rss"
327	+
328	+ ; This setting is overridden in the UI, under "User Settings".
329	+ ; The date and period loaded by Matomo uses the defaults below. Possible values: yesterday, today.
330	+ default_day = yesterday
331	+ ; Possible values: day, week, month, year.
332	+ default_period = day
333	+
334	+ ; Time in seconds after which an archive will be computed again. This setting is used only for today's statistics.
335	+ ; This setting is overriden in the UI, under "General Settings".
336	+ ; This setting is only used if it hasn't been overriden via the UI yet, or if enable_general_settings_admin=0
337	+ time_before_today_archive_considered_outdated = 900
338	+
339	+ ; Time in seconds after which an archive will be computed again. This setting is used only for week's statistics.
340	+ ; If set to "-1" (default), it will fall back to the UI setting under "General settings" unless enable_general_settings_admin=0
341	+ ; is set. In this case it will default to "time_before_today_archive_considered_outdated";
342	+ time_before_week_archive_considered_outdated = -1
343	+
344	+ ; Same as config setting "time_before_week_archive_considered_outdated" but it is only applied to monthly archives
345	+ time_before_month_archive_considered_outdated = -1
346	+
347	+ ; Same as config setting "time_before_week_archive_considered_outdated" but it is only applied to yearly archives
348	+ time_before_year_archive_considered_outdated = -1
349	+
350	+ ; Same as config setting "time_before_week_archive_considered_outdated" but it is only applied to range archives
351	+ time_before_range_archive_considered_outdated = -1
352	+
353	+ ; This setting is overriden in the UI, under "General Settings".
354	+ ; The default value is to allow browsers to trigger the Matomo archiving process.
355	+ ; This setting is only used if it hasn't been overridden via the UI yet, or if enable_general_settings_admin=0
356	+ enable_browser_archiving_triggering = 1
357	+
358	+ ; By default, Matomo will force archiving of range periods from browser requests, even if enable_browser_archiving_triggering
359	+ ; is set to 0. This can sometimes create too much of a demand on system resources. Setting this option to 0 and
360	+ ; disabling browser trigger archiving will make sure ranges are not archived on browser request. Since the cron
361	+ ; archiver does not archive any custom date ranges, you must either disable range (using enabled_periods_API and enabled_periods_UI)
362	+ ; or make sure the date ranges users' want to see will be processed somehow.
363	+ archiving_range_force_on_browser_request = 1
364	+
365	+ ; By default Matomo will automatically archive all date ranges any user has chosen in his account settings.
366	+ ; This is limited to the available options last7, previous7, last30 and previous30.
367	+ ; If you need any other period, or want to ensure one of those is always archived, you can define them here
368	+ archiving_custom_ranges[] =
369	+
370	+ ; By default Matomo runs OPTIMIZE TABLE SQL queries to free spaces after deleting some data.
371	+ ; If your Matomo tracks millions of pages, the OPTIMIZE TABLE queries might run for hours (seen in "SHOW FULL PROCESSLIST \g")
372	+ ; so you can disable these special queries here:
373	+ enable_sql_optimize_queries = 1
374	+
375	+ ; By default Matomo is purging complete date range archives to free spaces after deleting some data.
376	+ ; If you are pre-processing custom ranges using CLI task to make them easily available in UI,
377	+ ; you can prevent this action from happening by setting this parameter to value bigger than 1
378	+ purge_date_range_archives_after_X_days = 1
379	+
380	+ ; MySQL minimum required version
381	+ ; note: timezone support added in 4.1.3
382	+ minimum_mysql_version = 4.1
383	+
384	+ ; PostgreSQL minimum required version
385	+ minimum_pgsql_version = 8.3
386	+
387	+ ; Minimum advised memory limit in Mb in php.ini file (see memory_limit value)
388	+ ; Set to "-1" to always use the configured memory_limit value in php.ini file.
389	+ minimum_memory_limit = 128
390	+
391	+ ; Minimum memory limit in Mb enforced when archived via ./console core:archive
392	+ ; Set to "-1" to always use the configured memory_limit value in php.ini file.
393	+ minimum_memory_limit_when_archiving = 768
394	+
395	+ ; Matomo will check that usernames and password have a minimum length, and will check that characters are "allowed"
396	+ ; This can be disabled, if for example you wish to import an existing User database in Matomo and your rules are less restrictive
397	+ disable_checks_usernames_attributes = 0
398	+
399	+ ; Matomo will use the configured hash algorithm where possible.
400	+ ; For legacy data, fallback or non-security scenarios, we use md5.
401	+ hash_algorithm = whirlpool
402	+
403	+ ; Matomo uses PHP's dbtable for session.
404	+ ; If you prefer configuring sessions through the php.ini directly, you may unset this value to an empty string
405	+ session_save_handler = dbtable
406	+
407	+ ; If set to 1, Matomo will automatically redirect all http:// requests to https://
408	+ ; If SSL / https is not correctly configured on the server, this will break Matomo
409	+ ; If you set this to 1, and your SSL configuration breaks later on, you can always edit this back to 0
410	+ ; it is recommended for security reasons to always use Matomo over https
411	+ force_ssl = 0
412	+
413	+ ; (DEPRECATED) has no effect
414	+ login_cookie_name = piwik_auth
415	+
416	+ ; By default, the auth cookie is set only for the duration of session.
417	+ ; if "Remember me" is checked, the auth cookie will be valid for 14 days by default
418	+ login_cookie_expire = 1209600
419	+
420	+ ; Sets the session cookie path
421	+ login_cookie_path =
422	+
423	+ ; the amount of time before an idle session is considered expired. only affects session that were created without the
424	+ ; "remember me" option checked
425	+ login_session_not_remembered_idle_timeout = 3600
426	+
427	+ ; email address that appears as a Sender in the password recovery email
428	+ ; if specified, {DOMAIN} will be replaced by the current Matomo domain
429	+ login_password_recovery_email_address = "password-recovery@{DOMAIN}"
430	+ ; name that appears as a Sender in the password recovery email
431	+ login_password_recovery_email_name = Matomo
432	+
433	+ ; email address that appears as a Reply-to in the password recovery email
434	+ ; if specified, {DOMAIN} will be replaced by the current Matomo domain
435	+ login_password_recovery_replyto_email_address = "no-reply@{DOMAIN}"
436	+ ; name that appears as a Reply-to in the password recovery email
437	+ login_password_recovery_replyto_email_name = "No-reply"
438	+
439	+ ; When configured, only users from a configured IP can log into your Matomo. You can define one or multiple
440	+ ; IPv4, IPv6, and IP ranges. You may also define hostnames. However, resolving hostnames in each request
441	+ ; may slightly slow down your Matomo.
442	+ ; This whitelist also affects API requests unless you disabled it via the setting
443	+ ; "login_whitelist_apply_to_reporting_api_requests" below. Note that neither this setting, nor the
444	+ ; "login_whitelist_apply_to_reporting_api_requests" restricts authenticated tracking requests (tracking requests
445	+ ; with a "token_auth" URL parameter).
446	+ ;
447	+ ; Examples:
448	+ ; login_whitelist_ip[] = 204.93.240.*
449	+ ; login_whitelist_ip[] = 204.93.177.0/24
450	+ ; login_whitelist_ip[] = 199.27.128.0/21
451	+ ; login_whitelist_ip[] = 2001:db8::/48
452	+ ; login_whitelist_ip[] = matomo.org
453	+
454	+ ; By default, if a whitelisted IP address is specified via "login_whitelist_ip[]", the reporting user interface as
455	+ ; well as HTTP Reporting API requests will only work for these whitelisted IPs.
456	+ ; Set this setting to "0" to allow HTTP Reporting API requests from any IP address.
457	+ login_whitelist_apply_to_reporting_api_requests = 1
458	+
459	+ ; By default when user logs out they are redirected to Matomo "homepage" usually the Login form.
460	+ ; Uncomment the next line to set a URL to redirect the user to after they log out of Matomo.
461	+ ; login_logout_url = http://...
462	+
463	+ ; Set to 1 to disable the framebuster on standard Non-widgets pages (a click-jacking countermeasure).
464	+ ; Default is 0 (i.e., bust frames on all non Widget pages such as Login, API, Widgets, Email reports, etc.).
465	+ enable_framed_pages = 0
466	+
467	+ ; Set to 1 to disable the framebuster on Admin pages (a click-jacking countermeasure).
468	+ ; Default is 0 (i.e., bust frames on the Settings forms).
469	+ enable_framed_settings = 0
470	+
471	+ ; language cookie name for session
472	+ language_cookie_name = matomo_lang
473	+
474	+ ; standard email address displayed when sending emails
475	+ noreply_email_address = "noreply@{DOMAIN}"
476	+
477	+ ; standard email name displayed when sending emails. If not set, a default name will be used.
478	+ noreply_email_name = ""
479	+
480	+ ; set to 0 to disable sending of all emails. useful for testing.
481	+ emails_enabled = 1
482	+
483	+ ; set to 0 to disable sending of emails when a password or email is changed
484	+ enable_update_users_email = 1
485	+
486	+ ; feedback email address;
487	+ ; when testing, use your own email address or "nobody"
488	+ feedback_email_address = "feedback@matomo.org"
489	+
490	+ ; using to set reply_to in reports e-mail to login of report creator
491	+ scheduled_reports_replyto_is_user_email_and_alias = 0
492	+
493	+ ; scheduled reports truncate limit
494	+ ; the report will be rendered with the first 23 rows and will aggregate other rows in a summary row
495	+ ; 23 rows table fits in one portrait page
496	+ scheduled_reports_truncate = 23
497	+
498	+ ; during archiving, Matomo will limit the number of results recorded, for performance reasons
499	+ ; maximum number of rows for any of the Referrers tables (keywords, search engines, campaigns, etc.)
500	+ datatable_archiving_maximum_rows_referrers = 1000
501	+ ; maximum number of rows for any of the Referrers subtable (search engines by keyword, keyword by campaign, etc.)
502	+ datatable_archiving_maximum_rows_subtable_referrers = 50
503	+
504	+ ; maximum number of rows for the Users report
505	+ datatable_archiving_maximum_rows_userid_users = 50000
506	+
507	+ ; maximum number of rows for the Custom Variables names report
508	+ ; Note: if the website is Ecommerce enabled, the two values below will be automatically set to 50000
509	+ datatable_archiving_maximum_rows_custom_variables = 1000
510	+ ; maximum number of rows for the Custom Variables values reports
511	+ datatable_archiving_maximum_rows_subtable_custom_variables = 1000
512	+
513	+ ; maximum number of rows for any of the Actions tables (pages, downloads, outlinks)
514	+ datatable_archiving_maximum_rows_actions = 500
515	+ ; maximum number of rows for pages in categories (sub pages, when clicking on the + for a page category)
516	+ ; note: should not exceed the display limit in Piwik\Actions\Controller::ACTIONS_REPORT_ROWS_DISPLAY
517	+ ; because each subdirectory doesn't have paging at the bottom, so all data should be displayed if possible.
518	+ datatable_archiving_maximum_rows_subtable_actions = 100
519	+ ; maximum number of rows for the Site Search table
520	+ datatable_archiving_maximum_rows_site_search = 500
521	+
522	+ ; maximum number of rows for any of the Events tables (Categories, Actions, Names)
523	+ datatable_archiving_maximum_rows_events = 500
524	+ ; maximum number of rows for sub-tables of the Events tables (eg. for the subtables Categories>Actions or Categories>Names).
525	+ datatable_archiving_maximum_rows_subtable_events = 500
526	+
527	+ ; maximum number of rows for other tables (Providers, User settings configurations)
528	+ datatable_archiving_maximum_rows_standard = 500
529	+
530	+ ; maximum number of rows to fetch from the database when archiving. if set to 0, no limit is used.
531	+ ; this can be used to speed up the archiving process, but is only useful if you're site has a large
532	+ ; amount of actions, referrers or custom variable name/value pairs.
533	+ archiving_ranking_query_row_limit = 50000
534	+
535	+ ; maximum number of actions that is shown in the visitor log for each visitor
536	+ visitor_log_maximum_actions_per_visit = 500
537	+
538	+ ; by default, the real time Live! widget will update every 5 seconds and refresh with new visits/actions/etc.
539	+ ; you can change the timeout so the widget refreshes more often, or not as frequently
540	+ live_widget_refresh_after_seconds = 5
541	+
542	+ ; by default, the Live! real time visitor count widget will check to see how many visitors your
543	+ ; website received in the last 3 minutes. changing this value will change the number of minutes
544	+ ; the widget looks in.
545	+ live_widget_visitor_count_last_minutes = 3
546	+
547	+ ; by default visitor profile will show aggregated information for the last up to 100 visits of a visitor
548	+ ; this limit can be adjusted by changing this value
549	+ live_visitor_profile_max_visits_to_aggregate = 100
550	+
551	+ ; If configured, will abort a MySQL query after the configured amount of seconds and show an error in the UI to for
552	+ ; example lower the date range or tweak the segment (if one is applied). Set it to -1 if the query time should not be
553	+ ; limited. Note: This feature requires a recent MySQL version (5.7 or newer). Some MySQL forks like MariaDB might not
554	+ ; support this feature which uses the MAX_EXECUTION_TIME hint.
555	+ live_query_max_execution_time = -1
556	+
557	+ ; In "All Websites" dashboard, when looking at today's reports (or a date range including today),
558	+ ; the page will automatically refresh every 5 minutes. Set to 0 to disable automatic refresh
559	+ multisites_refresh_after_seconds = 300
560	+
561	+ ; by default, an update notification for a new version of Matomo is shown to every user. Set to 1 if only
562	+ ; the superusers should see the notification.
563	+ show_update_notification_to_superusers_only = 0
564	+
565	+ ; Set to 1 if you're using https on your Matomo server and Matomo can't detect it,
566	+ ; e.g., a reverse proxy using https-to-http, or a web server that doesn't
567	+ ; set the HTTPS environment variable.
568	+ assume_secure_protocol = 0
569	+
570	+ ; Set to 1 if you're using more than one server for your Matomo installation. For example if you are using Matomo in a
571	+ ; load balanced environment, if you have configured failover or if you're just using multiple servers in general.
572	+ ; By enabling this flag we will for example not allow the installation of a plugin via the UI as a plugin would be only
573	+ ; installed on one server or a config one change would be only made on one server instead of all servers.
574	+ multi_server_environment = 0
575	+
576	+ ; List of proxy headers for client IP addresses
577	+ ; Matomo will determine the user IP by extracting the first IP address found in this proxy header.
578	+ ;
579	+ ; CloudFlare (CF-Connecting-IP)
580	+ ;proxy_client_headers[] = HTTP_CF_CONNECTING_IP
581	+ ;
582	+ ; ISP proxy (Client-IP)
583	+ ;proxy_client_headers[] = HTTP_CLIENT_IP
584	+ ;
585	+ ; de facto standard (X-Forwarded-For)
586	+ ;proxy_client_headers[] = HTTP_X_FORWARDED_FOR
587	+
588	+ ; List of proxy headers for host IP addresses
589	+ ;
590	+ ; de facto standard (X-Forwarded-Host)
591	+ ;proxy_host_headers[] = HTTP_X_FORWARDED_HOST
592	+
593	+ ; List of proxy IP addresses (or IP address ranges) to skip (if present in the above headers).
594	+ ; Generally, only required if there's more than one proxy between the visitor and the backend web server.
595	+ ;
596	+ ; Examples:
597	+ ;proxy_ips[] = 204.93.240.*
598	+ ;proxy_ips[] = 204.93.177.0/24
599	+ ;proxy_ips[] = 199.27.128.0/21
600	+ ;proxy_ips[] = 173.245.48.0/20
601	+
602	+ ; Set to 1 if you're using a proxy which is rewriting the URI.
603	+ ; By enabling this flag the header HTTP_X_FORWARDED_URI will be considered for the current script name.
604	+ proxy_uri_header = 0
605	+
606	+ ; Whether to enable trusted host checking. This can be disabled if you're running Matomo
607	+ ; on several URLs and do not wish to constantly edit the trusted host list.
608	+ enable_trusted_host_check = 1
609	+
610	+ ; List of trusted hosts (eg domain or subdomain names) when generating absolute URLs.
611	+ ;
612	+ ; Examples:
613	+ ;trusted_hosts[] = example.com
614	+ ;trusted_hosts[] = stats.example.com
615	+
616	+ ; List of Cross-origin resource sharing domains (eg domain or subdomain names) when generating absolute URLs.
617	+ ; Described here: https://en.wikipedia.org/wiki/Cross-origin_resource_sharing
618	+ ;
619	+ ; Examples:
620	+ ;cors_domains[] = http://example.com
621	+ ;cors_domains[] = http://stats.example.com
622	+ ;
623	+ ; Or you may allow cross domain requests for all domains with:
624	+ ;cors_domains[] = *
625	+
626	+ ; If you use this Matomo instance over multiple hostnames, Matomo will need to know
627	+ ; a unique instance_id for this instance, so that Matomo can serve the right custom logo and tmp/* assets,
628	+ ; independently of the hostname Matomo is currently running under.
629	+ ; instance_id = stats.example.com
630	+
631	+ ; The API server is an essential part of the Matomo infrastructure/ecosystem to
632	+ ; provide services to Matomo installations, e.g., getLatestVersion and
633	+ ; subscribeNewsletter.
634	+ api_service_url = http://api.matomo.org
635	+
636	+ ; When the ImageGraph plugin is activated, report metadata have an additional entry : 'imageGraphUrl'.
637	+ ; This entry can be used to request a static graph for the requested report.
638	+ ; When requesting report metadata with $period=range, Matomo needs to translate it to multiple periods for evolution graphs.
639	+ ; eg. $period=range&date=previous10 becomes $period=day&date=previous10. Use this setting to override the $period value.
640	+ graphs_default_period_to_plot_when_period_range = day
641	+
642	+ ; When the ImageGraph plugin is activated, enabling this option causes the image graphs to show the evolution
643	+ ; within the selected period instead of the evolution across the last n periods.
644	+ graphs_show_evolution_within_selected_period = 0
645	+
646	+ ; This option controls the default number of days in the past to show in evolution graphs generated by the ImageGraph plugin
647	+ graphs_default_evolution_graph_last_days_amount = 30
648	+
649	+ ; The Overlay plugin shows the Top X following pages, Top X downloads and Top X outlinks which followed
650	+ ; a view of the current page. The value X can be set here.
651	+ overlay_following_pages_limit = 300
652	+
653	+ ; With this option, you can disable the framed mode of the Overlay plugin. Use it if your website contains a framebuster.
654	+ overlay_disable_framed_mode = 0
655	+
656	+ ; Controls whether the user is able to upload a custom logo for their Matomo install
657	+ enable_custom_logo = 1
658	+
659	+ ; By default we check whether the Custom logo is writable or not, before we display the Custom logo file uploader
660	+ enable_custom_logo_check = 1
661	+
662	+ ; If php is running in a chroot environment, when trying to import CSV files with createTableFromCSVFile(),
663	+ ; Mysql will try to load the chrooted path (which is incomplete). To prevent an error, here you can specify the
664	+ ; absolute path to the chroot environment. eg. '/path/to/matomo/chrooted/'
665	+ absolute_chroot_path =
666	+
667	+ ; The path (relative to the Matomo directory) in which Matomo temporary files are stored.
668	+ ; Defaults to ./tmp (the tmp/ folder inside the Matomo directory)
669	+ tmp_path = "/tmp"
670	+
671	+ ; In some rare cases it may be useful to explicitely tell Matomo not to use LOAD DATA INFILE
672	+ ; This may for example be useful when doing Mysql AWS replication
673	+ enable_load_data_infile = 1
674	+
675	+ ; By setting this option to 0:
676	+ ; - links to Enable/Disable/Uninstall plugins will be hidden and disabled
677	+ ; - links to Uninstall themes will be disabled (but user can still enable/disable themes)
678	+ enable_plugins_admin = 1
679	+
680	+ ; By setting this option to 0 the users management will be disabled
681	+ enable_users_admin = 1
682	+
683	+ ; By setting this option to 0 the websites management will be disabled
684	+ enable_sites_admin = 1
685	+
686	+ ; By setting this option to 1, it will be possible for Super Users to upload Matomo plugin ZIP archives directly in Matomo Administration.
687	+ ; Enabling this opens a remote code execution vulnerability where
688	+ ; an attacker who gained Super User access could execute custom PHP code in a Matomo plugin.
689	+ enable_plugin_upload = 0
690	+
691	+ ; By setting this option to 0 (e.g. in common.config.ini.php) the installer will be disabled.
692	+ enable_installer = 1
693	+
694	+ ; By setting this option to 0, you can prevent Super User from editing the Geolocation settings.
695	+ enable_geolocation_admin = 1
696	+
697	+ ; By setting this option to 0, the old raw data and old report data purging features will be hidden from the UI
698	+ ; Note: log purging and old data purging still occurs, just the Super User cannot change the settings.
699	+ enable_delete_old_data_settings_admin = 1
700	+
701	+ ; By setting this option to 0, the following settings will be hidden and disabled from being set in the UI:
702	+ ; - Archiving settings
703	+ ; - Update settings
704	+ ; - Email server settings
705	+ ; - Trusted Matomo Hostname
706	+ enable_general_settings_admin = 1
707	+
708	+ ; Disabling this will disable features like automatic updates for Matomo,
709	+ ; its plugins and components like the GeoIP database, referrer spam blacklist or search engines and social network definitions
710	+ enable_internet_features = 1
711	+
712	+ ; By setting this option to 0, it will disable the "Auto update" feature
713	+ enable_auto_update = 1
714	+
715	+ ; By setting this option to 0, no emails will be sent in case of an available core.
716	+ ; If set to 0 it also disables the "sent plugin update emails" feature in general and the related setting in the UI.
717	+ enable_update_communication = 1
718	+
719	+ ; Comma separated list of plugin names for which console commands should be loaded (applies when Matomo is not installed yet)
720	+ always_load_commands_from_plugin=
721	+
722	+ ; This controls whether the pivotBy query parameter can be used with any dimension or just subtable
723	+ ; dimensions. If set to 1, it will fetch a report with a segment for each row of the table being pivoted.
724	+ ; At present, this is very inefficient, so it is disabled by default.
725	+ pivot_by_filter_enable_fetch_by_segment = 0
726	+
727	+ ; This controls the default maximum number of columns to display in a pivot table. Since a pivot table displays
728	+ ; a table's rows as columns, the number of columns can become very large, which will affect webpage layouts.
729	+ ; Set to -1 to specify no limit. Note: The pivotByColumnLimit query parameter can be used to override this default
730	+ ; on a per-request basis;
731	+ pivot_by_filter_default_column_limit = 10
732	+
733	+ ; If set to 0 it will disable advertisements for providers of Professional Support for Matomo.
734	+ piwik_professional_support_ads_enabled = 1
735	+
736	+ ; The number of days to wait before sending the JavaScript tracking code email reminder.
737	+ num_days_before_tracking_code_reminder = 5
738	+
739	+ ; The maximum number of segments that can be compared simultaneously.
740	+ data_comparison_segment_limit = 5
741	+
742	+ ; The maximum number of periods that can be compared simultaneously.
743	+ data_comparison_period_limit = 5
744	+
745	+ ; The path to a custom cacert.pem file Matomo should use.
746	+ ; By default Matomo uses a file extracted from the Firefox browser and provided here: https://curl.haxx.se/docs/caextract.html.
747	+ ; The file contains root CAs and is used to determine if the chain of a SSL certificate is valid and it is safe to connect.
748	+ ; Most users will not have to use a custom file here, but if you run your Matomo instance behind a proxy server/firewall that
749	+ ; breaks and reencrypts SSL connections you can set your custom file here.
750	+ custom_cacert_pem=
751	+
752	+ ; Whether or not to send weekly emails to superusers about tracking failures.
753	+ ; Default is 1.
754	+ enable_tracking_failures_notification = 1
755	+
756	+ [Tracker]
757	+
758	+ ; Matomo uses "Privacy by default" model. When one of your users visit multiple of your websites tracked in this Matomo,
759	+ ; Matomo will create for this user a fingerprint that will be different across the multiple websites.
760	+ ; If you want to track unique users across websites you may set this setting to 1.
761	+ ; Note: setting this to 0 increases your users' privacy.
762	+ enable_fingerprinting_across_websites = 0
763	+
764	+ ; Matomo uses first party cookies by default. If set to 1,
765	+ ; the visit ID cookie will be set on the Matomo server domain as well
766	+ ; this is useful when you want to do cross websites analysis
767	+ use_third_party_id_cookie = 0
768	+
769	+ ; If tracking does not work for you or you are stuck finding an issue, you might want to enable the tracker debug mode.
770	+ ; Once enabled (set to 1) messages will be logged to all loggers defined in "[log] log_writers" config.
771	+ debug = 0
772	+
773	+ ; This option is an alternative to the debug option above. When set to 1, you can debug tracker request by adding
774	+ ; a debug=1 query parameter in the URL. All other HTTP requests will not have debug enabled. For security reasons this
775	+ ; option should be only enabled if really needed and only for a short time frame. Otherwise anyone can set debug=1 and
776	+ ; see the log output as well.
777	+ debug_on_demand = 0
778	+
779	+ ; This setting is described in this FAQ: https://matomo.org/faq/how-to/faq_175/
780	+ ; Note: generally this should only be set to 1 in an intranet setting, where most users have the same configuration (browsers, OS)
781	+ ; and the same IP. If left to 0 in this setting, all visitors will be counted as one single visitor.
782	+ trust_visitors_cookies = 0
783	+
784	+ ; name of the cookie used to store the visitor information
785	+ ; This is used only if use_third_party_id_cookie = 1
786	+ cookie_name = _pk_uid
787	+
788	+ ; by default, the Matomo tracking cookie expires in 13 months (365 + 28 days)
789	+ ; This is used only if use_third_party_id_cookie = 1
790	+ cookie_expire = 33955200;
791	+
792	+ ; The path on the server in which the cookie will be available on.
793	+ ; Defaults to empty. See spec in https://curl.haxx.se/rfc/cookie_spec.html
794	+ ; This is used for the Ignore cookie, and the third party cookie if use_third_party_id_cookie = 1
795	+ cookie_path =
796	+
797	+ ; The domain on the server in which the cookie will be available on.
798	+ ; Defaults to empty. See spec in https://curl.haxx.se/rfc/cookie_spec.html
799	+ ; This is used for the third party cookie if use_third_party_id_cookie = 1
800	+ cookie_domain =
801	+
802	+ ; set to 0 if you want to stop tracking the visitors. Useful if you need to stop all the connections on the DB.
803	+ record_statistics = 1
804	+
805	+ ; length of a visit in seconds. If a visitor comes back on the website visit_standard_length seconds
806	+ ; after their last page view, it will be recorded as a new visit. In case you are using the Matomo JavaScript tracker to
807	+ ; calculate the visit count correctly, make sure to call the method "setSessionCookieTimeout" eg
808	+ ; `_paq.push(['setSessionCookieTimeout', timeoutInSeconds=1800])`
809	+ visit_standard_length = 1800
810	+
811	+ ; The amount of time in the past to match the current visitor to a known visitor via fingerprint. Defaults to visit_standard_length.
812	+ ; If you are looking for higher accuracy of "returning visitors" metrics, you may set this value to 86400 or more.
813	+ ; This is especially useful when you use the Tracking API where tracking Returning Visitors often depends on this setting.
814	+ ; The value window_look_back_for_visitor is used only if it is set to greater than visit_standard_length.
815	+ ; Note: visitors with visitor IDs will be matched by visitor ID from any point in time, this is only for recognizing visitors
816	+ ; by device fingerprint.
817	+ window_look_back_for_visitor = 0
818	+
819	+ ; visitors that stay on the website and view only one page will be considered as time on site of 0 second
820	+ default_time_one_page_visit = 0
821	+
822	+ ; Comma separated list of URL query string variable names that will be removed from your tracked URLs
823	+ ; By default, Matomo will remove the most common parameters which are known to change often (eg. session ID parameters)
824	+ url_query_parameter_to_exclude_from_url = "gclid,fbclid,fb_xd_fragment,fb_comment_id,phpsessid,jsessionid,sessionid,aspsessionid,doing_wp_cron,sid,pk_vid"
825	+
826	+ ; if set to 1, Matomo attempts a "best guess" at the visitor's country of
827	+ ; origin when the preferred language tag omits region information.
828	+ ; The mapping is defined in core/DataFiles/LanguageToCountry.php,
829	+ enable_language_to_country_guess = 1
830	+
831	+ ; When the `./console core:archive` cron hasn't been setup, we still need to regularly run some maintenance tasks.
832	+ ; Visits to the Tracker will try to trigger Scheduled Tasks (eg. scheduled PDF/HTML reports by email).
833	+ ; Scheduled tasks will only run if 'Enable Matomo Archiving from Browser' is enabled in the General Settings.
834	+ ; Tasks run once every hour maximum, they might not run every hour if traffic is low.
835	+ ; Set to 0 to disable Scheduled tasks completely.
836	+ scheduled_tasks_min_interval = 3600
837	+
838	+ ; name of the cookie to ignore visits
839	+ ignore_visits_cookie_name = piwik_ignore
840	+
841	+ ; Comma separated list of variable names that will be read to define a Campaign name, for example CPC campaign
842	+ ; Example: If a visitor first visits 'index.php?piwik_campaign=Adwords-CPC' then it will be counted as a campaign referrer named 'Adwords-CPC'
843	+ ; Includes by default the GA style campaign parameters
844	+ campaign_var_name = "pk_cpn,pk_campaign,piwik_campaign,utm_campaign,utm_source,utm_medium"
845	+
846	+ ; Comma separated list of variable names that will be read to track a Campaign Keyword
847	+ ; Example: If a visitor first visits 'index.php?piwik_campaign=Adwords-CPC&piwik_kwd=My killer keyword' ;
848	+ ; then it will be counted as a campaign referrer named 'Adwords-CPC' with the keyword 'My killer keyword'
849	+ ; Includes by default the GA style campaign keyword parameter utm_term
850	+ campaign_keyword_var_name = "pk_kwd,pk_keyword,piwik_kwd,utm_term"
851	+
852	+ ; if set to 1, actions that contain different campaign information from the visitor's ongoing visit will
853	+ ; be treated as the start of a new visit. This will include situations when campaign information was absent before,
854	+ ; but is present now.
855	+ create_new_visit_when_campaign_changes = 1
856	+
857	+ ; if set to 1, actions that contain different website referrer information from the visitor's ongoing visit
858	+ ; will be treated as the start of a new visit. This will include situations when website referrer information was
859	+ ; absent before, but is present now.
860	+ create_new_visit_when_website_referrer_changes = 0
861	+
862	+ ; ONLY CHANGE THIS VALUE WHEN YOU DO NOT USE MATOMO ARCHIVING, SINCE THIS COULD CAUSE PARTIALLY MISSING ARCHIVE DATA
863	+ ; Whether to force a new visit at midnight for every visitor. Default 1.
864	+ create_new_visit_after_midnight = 1
865	+
866	+ ; maximum length of a Page Title or a Page URL recorded in the log_action.name table
867	+ page_maximum_length = 1024;
868	+
869	+ ; Tracker cache files are the simple caching layer for Tracking.
870	+ ; TTL: Time to live for cache files, in seconds. Default to 5 minutes.
871	+ tracker_cache_file_ttl = 300
872	+
873	+ ; Whether Bulk tracking requests to the Tracking API requires the token_auth to be set.
874	+ bulk_requests_require_authentication = 0
875	+
876	+ ; Whether Bulk tracking requests will be wrapped within a DB Transaction.
877	+ ; This greatly increases performance of Log Analytics and in general any Bulk Tracking API requests.
878	+ bulk_requests_use_transaction = 1
879	+
880	+ ; DO NOT USE THIS SETTING ON PUBLICLY AVAILABLE MATOMO SERVER
881	+ ; !!! Security risk: if set to 0, it would allow anyone to push data to Matomo with custom dates in the past/future and even with fake IPs!
882	+ ; When using the Tracking API, to override either the datetime and/or the visitor IP,
883	+ ; token_auth with an "admin" access is required. If you set this setting to 0, the token_auth will not be required anymore.
884	+ ; DO NOT USE THIS SETTING ON PUBLIC MATOMO SERVERS
885	+ tracking_requests_require_authentication = 1
886	+
887	+ ; By default, Matomo accepts only tracking requests for up to 1 day in the past. For tracking requests with a custom date
888	+ ; date is older than 1 day, Matomo requires an authenticated tracking requests. By setting this config to another value
889	+ ; You can change how far back Matomo will track your requests without authentication. The configured value is in seconds.
890	+ tracking_requests_require_authentication_when_custom_timestamp_newer_than = 86400;
891	+
892	+ ; if set to 1, all the SQL queries will be recorded by the profiler
893	+ ; and a profiling summary will be printed at the end of the request
894	+ ; NOTE: you must also set "[Tracker] debug = 1" to enable the profiler.
895	+ enable_sql_profiler = 0
896	+
897	+ [Segments]
898	+ ; Reports with segmentation in API requests are processed in real time.
899	+ ; On high traffic websites it is recommended to pre-process the data
900	+ ; so that the analytics reports are always fast to load.
901	+ ; You can define below the list of Segments strings
902	+ ; for which all reports should be Archived during the cron execution
903	+ ; All segment values MUST be URL encoded.
904	+ ;Segments[]="visitorType==new"
905	+ ;Segments[]="visitorType==returning,visitorType==returningCustomer"
906	+
907	+ ; If you define Custom Variables for your visitor, for example set the visit type
908	+ ;Segments[]="customVariableName1==VisitType;customVariableValue1==Customer"
909	+
910	+ [Deletelogs]
911	+ ; delete_logs_enable - enable (1) or disable (0) delete log feature. Make sure that all archives for the given period have been processed (setup a cronjob!),
912	+ ; otherwise you may lose tracking data.
913	+ ; delete_logs_schedule_lowest_interval - lowest possible interval between two table deletes (in days, 1\|7\|30). Default: 7.
914	+ ; delete_logs_older_than - delete data older than XX (days). Default: 180
915	+ delete_logs_enable = 0
916	+ delete_logs_schedule_lowest_interval = 7
917	+ delete_logs_older_than = 180
918	+ delete_logs_max_rows_per_query = 100000
919	+ enable_auto_database_size_estimate = 1
920	+ enable_database_size_estimate = 1
921	+ delete_logs_unused_actions_schedule_lowest_interval = 30
922	+
923	+ [Deletereports]
924	+ delete_reports_enable = 0
925	+ delete_reports_older_than = 12
926	+ delete_reports_keep_basic_metrics = 1
927	+ delete_reports_keep_day_reports = 0
928	+ delete_reports_keep_week_reports = 0
929	+ delete_reports_keep_month_reports = 1
930	+ delete_reports_keep_year_reports = 1
931	+ delete_reports_keep_range_reports = 0
932	+ delete_reports_keep_segment_reports = 0
933	+
934	+ [mail]
935	+ defaultHostnameIfEmpty = defaultHostnameIfEmpty.example.org ; default Email @hostname, if current host can't be read from system variables
936	+ transport = ; smtp (using the configuration below) or empty (using built-in mail() function)
937	+ port = ; optional; defaults to 25 when security is none or tls; 465 for ssl
938	+ host = ; SMTP server address
939	+ type = ; SMTP Auth type. By default: NONE. For example: LOGIN
940	+ username = ; SMTP username
941	+ password = ; SMTP password
942	+ encryption = ; SMTP transport-layer encryption, either 'ssl', 'tls', or empty (i.e., none).
943	+
944	+ [proxy]
945	+ type = BASIC ; proxy type for outbound/outgoing connections; currently, only BASIC is supported
946	+ host = ; Proxy host: the host name of your proxy server (mandatory)
947	+ port = ; Proxy port: the port that the proxy server listens to. There is no standard default, but 80, 1080, 3128, and 8080 are popular
948	+ username = ; Proxy username: optional; if specified, password is mandatory
949	+ password = ; Proxy password: optional; if specified, username is mandatory
950	+
951	+ [Plugins]
952	+ ; list of plugins (in order they will be loaded) that are activated by default in the Matomo platform
953	+ Plugins[] = CorePluginsAdmin
954	+ Plugins[] = CoreAdminHome
955	+ Plugins[] = CoreHome
956	+ Plugins[] = WebsiteMeasurable
957	+ Plugins[] = IntranetMeasurable
958	+ Plugins[] = Diagnostics
959	+ Plugins[] = CoreVisualizations
960	+ Plugins[] = Proxy
961	+ Plugins[] = API
962	+ Plugins[] = Widgetize
963	+ Plugins[] = Transitions
964	+ Plugins[] = LanguagesManager
965	+ Plugins[] = Actions
966	+ Plugins[] = Dashboard
967	+ Plugins[] = MultiSites
968	+ Plugins[] = Referrers
969	+ Plugins[] = UserLanguage
970	+ Plugins[] = DevicesDetection
971	+ Plugins[] = Goals
972	+ Plugins[] = Ecommerce
973	+ Plugins[] = SEO
974	+ Plugins[] = Events
975	+ Plugins[] = UserCountry
976	+ Plugins[] = GeoIp2
977	+ Plugins[] = VisitsSummary
978	+ Plugins[] = VisitFrequency
979	+ Plugins[] = VisitTime
980	+ Plugins[] = VisitorInterest
981	+ Plugins[] = RssWidget
982	+ Plugins[] = Feedback
983	+ Plugins[] = Monolog
984	+
985	+ Plugins[] = Login
986	+ Plugins[] = TwoFactorAuth
987	+ Plugins[] = UsersManager
988	+ Plugins[] = SitesManager
989	+ Plugins[] = Installation
990	+ Plugins[] = CoreUpdater
991	+ Plugins[] = CoreConsole
992	+ Plugins[] = ScheduledReports
993	+ Plugins[] = UserCountryMap
994	+ Plugins[] = Live
995	+ Plugins[] = CustomVariables
996	+ Plugins[] = PrivacyManager
997	+ Plugins[] = ImageGraph
998	+ Plugins[] = Annotations
999	+ Plugins[] = MobileMessaging
1000	+ Plugins[] = Overlay
1001	+ Plugins[] = SegmentEditor
1002	+ Plugins[] = Insights
1003	+ Plugins[] = Morpheus
1004	+ Plugins[] = Contents
1005	+ Plugins[] = BulkTracking
1006	+ Plugins[] = Resolution
1007	+ Plugins[] = DevicePlugins
1008	+ Plugins[] = Heartbeat
1009	+ Plugins[] = Intl
1010	+ Plugins[] = Marketplace
1011	+ Plugins[] = ProfessionalServices
1012	+ Plugins[] = UserId
1013	+ Plugins[] = CustomPiwikJs
1014	+ Plugins[] = Tour
1015	+
1016	+ [PluginsInstalled]
1017	+ PluginsInstalled[] = Diagnostics
1018	+ PluginsInstalled[] = Login
1019	+ PluginsInstalled[] = CoreAdminHome
1020	+ PluginsInstalled[] = UsersManager
1021	+ PluginsInstalled[] = SitesManager
1022	+ PluginsInstalled[] = Installation
1023	+ PluginsInstalled[] = Monolog
1024	+ PluginsInstalled[] = Intl
1025	+
1026	+ [APISettings]
1027	+ ; Any key/value pair can be added in this section, they will be available via the REST call
1028	+ ; index.php?module=API&method=API.getSettings
1029	+ ; This can be used to expose values from Matomo, to control for example a Mobile app tracking
1030	+ SDK_batch_size = 10
1031	+ SDK_interval_value = 30
1032	+
1033	+ ; NOTE: do not directly edit this file! See notice at the top

app/config/global.php ADDED Viewed

	@@ -0,0 +1,221 @@


1	+ <?php
2	+
3	+ use Interop\Container\ContainerInterface;
4	+ use Interop\Container\Exception\NotFoundException;
5	+ use Piwik\Cache\Eager;
6	+ use Piwik\SettingsServer;
7	+ use Piwik\Config;
8	+
9	+ return array(
10	+
11	+ 'path.root' => PIWIK_DOCUMENT_ROOT,
12	+
13	+ 'path.misc.user' => 'misc/user/',
14	+
15	+ 'path.tmp' => function (ContainerInterface $c) {
16	+ $root = PIWIK_USER_PATH;
17	+
18	+ // TODO remove that special case and instead have plugins override 'path.tmp' to add the instance id
19	+ if ($c->has('ini.General.instance_id')) {
20	+ $instanceId = $c->get('ini.General.instance_id');
21	+ $instanceId = $instanceId ? '/' . $instanceId : '';
22	+ } else {
23	+ $instanceId = '';
24	+ }
25	+
26	+ /** @var Piwik\Config\ $config */
27	+ $config = $c->get('Piwik\Config');
28	+ $general = $config->General;
29	+ $tmp = empty($general['tmp_path']) ? '/tmp' : $general['tmp_path'];
30	+
31	+ return $root . $tmp . $instanceId;
32	+ },
33	+
34	+ 'path.cache' => DI\string('{path.tmp}/cache/tracker/'),
35	+
36	+ 'Piwik\Cache\Eager' => function (ContainerInterface $c) {
37	+ $backend = $c->get('Piwik\Cache\Backend');
38	+ $cacheId = $c->get('cache.eager.cache_id');
39	+
40	+ if (SettingsServer::isTrackerApiRequest()) {
41	+ $eventToPersist = 'Tracker.end';
42	+ $cacheId .= 'tracker';
43	+ } else {
44	+ $eventToPersist = 'Request.dispatch.end';
45	+ $cacheId .= 'ui';
46	+ }
47	+
48	+ $cache = new Eager($backend, $cacheId);
49	+ \Piwik\Piwik::addAction($eventToPersist, function () use ($cache) {
50	+ $cache->persistCacheIfNeeded(43200);
51	+ });
52	+
53	+ return $cache;
54	+ },
55	+ 'Piwik\Cache\Backend' => function (ContainerInterface $c) {
56	+ // If Piwik is not installed yet, it's possible the tmp/ folder is not writable
57	+ // we prevent failing with an unclear message eg. coming from doctrine-cache
58	+ // by forcing to use a cache backend which always works ie. array
59	+ if(!\Piwik\SettingsPiwik::isPiwikInstalled()) {
60	+ $backend = 'array';
61	+ } else {
62	+ try {
63	+ $backend = $c->get('ini.Cache.backend');
64	+ } catch (NotFoundException $ex) {
65	+ $backend = 'chained'; // happens if global.ini.php is not available
66	+ }
67	+ }
68	+
69	+ return \Piwik\Cache::buildBackend($backend);
70	+ },
71	+ 'cache.eager.cache_id' => function () {
72	+ return 'eagercache-' . str_replace(array('.', '-'), '', \Piwik\Version::VERSION) . '-';
73	+ },
74	+
75	+ 'entities.idNames' => DI\add(array('idGoal', 'idDimension')),
76	+
77	+ 'Psr\Log\LoggerInterface' => DI\object('Psr\Log\NullLogger'),
78	+
79	+ 'Piwik\Translation\Loader\LoaderInterface' => DI\object('Piwik\Translation\Loader\LoaderCache')
80	+ ->constructor(DI\get('Piwik\Translation\Loader\JsonFileLoader')),
81	+
82	+ 'observers.global' => array(),
83	+
84	+ /**
85	+ * By setting this option to false, the check that the DB schema version matches the version of the source code will be no longer performed.
86	+ * Thus it allows you to execute for example a newer version of Matomo with an older Matomo database version. Please note
87	+ * disabling this setting is not recommended because often an older DB version is not compatible with newer source code.
88	+ * If you disable this setting, make sure to execute the updates after updating the source code. The setting can be useful if
89	+ * you want to update Matomo without any outage when you know the current source code update will still run fine for a short time
90	+ * while in the background the database updates are running.
91	+ */
92	+ 'EnableDbVersionCheck' => true,
93	+
94	+ 'fileintegrity.ignore' => DI\add(array(
95	+ '*.htaccess',
96	+ '*web.config',
97	+ 'bootstrap.php',
98	+ 'favicon.ico',
99	+ 'robots.txt',
100	+ '.bowerrc',
101	+ '.lfsconfig',
102	+ '.phpstorm.meta.php',
103	+ 'config/config.ini.php',
104	+ 'config/config.php',
105	+ 'config/common.ini.php',
106	+ 'config/*.config.ini.php',
107	+ 'config/manifest.inc.php',
108	+ 'misc/*.dat',
109	+ 'misc/*.dat.gz',
110	+ 'misc/*.mmdb',
111	+ 'misc/*.mmdb.gz',
112	+ 'misc/*.bin',
113	+ 'misc/user/*png',
114	+ 'misc/user/*svg',
115	+ 'misc/user/*js',
116	+ 'misc/user/*/config.ini.php',
117	+ 'misc/package',
118	+ 'misc/package/WebAppGallery/*.xml',
119	+ 'misc/package/WebAppGallery/install.sql',
120	+ 'plugins/ImageGraph/fonts/unifont.ttf',
121	+ 'plugins/*/config/tracker.php',
122	+ 'plugins/*/config/config.php',
123	+ 'vendor/autoload.php',
124	+ 'vendor/composer/autoload_real.php',
125	+ 'vendor/szymach/c-pchart/app/*',
126	+ 'tmp/*',
127	+ // Search engine sites verification
128	+ 'google*.html',
129	+ 'BingSiteAuth.xml',
130	+ 'yandex*.html',
131	+ // common files on shared hosters
132	+ 'php.ini',
133	+ '.user.ini',
134	+ // Files below are not expected but they used to be present in older Piwik versions and may be still here
135	+ // As they are not going to cause any trouble we won't report them as 'File to delete'
136	+ '*.coveralls.yml',
137	+ '*.scrutinizer.yml',
138	+ '*.gitignore',
139	+ '*.gitkeep',
140	+ '*.gitmodules',
141	+ '*.gitattributes',
142	+ '*.bower.json',
143	+ '*.travis.yml',
144	+ )),
145	+
146	+ 'Piwik\EventDispatcher' => DI\object()->constructorParameter('observers', DI\get('observers.global')),
147	+
148	+ 'login.whitelist.ips' => function (ContainerInterface $c) {
149	+ /** @var Piwik\Config\ $config */
150	+ $config = $c->get('Piwik\Config');
151	+ $general = $config->General;
152	+
153	+ $ips = array();
154	+ if (!empty($general['login_whitelist_ip']) && is_array($general['login_whitelist_ip'])) {
155	+ $ips = $general['login_whitelist_ip'];
156	+ }
157	+
158	+ $ipsResolved = array();
159	+
160	+ foreach ($ips as $ip) {
161	+ if (filter_var($ip, FILTER_VALIDATE_IP)) {
162	+ $ipsResolved[] = $ip;
163	+ } else {
164	+ $ipFromHost = @gethostbyname($ip);
165	+ if (!empty($ipFromHost)) {
166	+ $ipsResolved[] = $ipFromHost;
167	+ }
168	+ }
169	+ }
170	+
171	+ return $ipsResolved;
172	+ },
173	+ 'Zend_Mail_Transport_Abstract' => function () {
174	+ $mailConfig = Config::getInstance()->mail;
175	+
176	+ if (empty($mailConfig['host'])
177	+ \|\| $mailConfig['transport'] != 'smtp'
178	+ ) {
179	+ return;
180	+ }
181	+
182	+ $smtpConfig = array();
183	+ if (!empty($mailConfig['type'])) {
184	+ $smtpConfig['auth'] = strtolower($mailConfig['type']);
185	+ }
186	+
187	+ if (!empty($mailConfig['username'])) {
188	+ $smtpConfig['username'] = $mailConfig['username'];
189	+ }
190	+
191	+ if (!empty($mailConfig['password'])) {
192	+ $smtpConfig['password'] = $mailConfig['password'];
193	+ }
194	+
195	+ if (!empty($mailConfig['encryption'])) {
196	+ $smtpConfig['ssl'] = $mailConfig['encryption'];
197	+ }
198	+
199	+ if (!empty($mailConfig['port'])) {
200	+ $smtpConfig['port'] = $mailConfig['port'];
201	+ }
202	+
203	+ $host = trim($mailConfig['host']);
204	+ $transport = new \Zend_Mail_Transport_Smtp($host, $smtpConfig);
205	+ return $transport;
206	+ },
207	+
208	+ 'Piwik\Tracker\VisitorRecognizer' => DI\object()
209	+ ->constructorParameter('trustCookiesOnly', DI\get('ini.Tracker.trust_visitors_cookies'))
210	+ ->constructorParameter('visitStandardLength', DI\get('ini.Tracker.visit_standard_length'))
211	+ ->constructorParameter('lookbackNSecondsCustom', DI\get('ini.Tracker.window_look_back_for_visitor')),
212	+
213	+ 'Piwik\Tracker\Settings' => DI\object()
214	+ ->constructorParameter('isSameFingerprintsAcrossWebsites', DI\get('ini.Tracker.enable_fingerprinting_across_websites')),
215	+
216	+ 'archiving.performance.logger' => null,
217	+
218	+ \Piwik\CronArchive\Performance\Logger::class => DI\object()->constructorParameter('logger', DI\get('archiving.performance.logger')),
219	+
220	+ \Piwik\Concurrency\LockBackend::class => \DI\get(\Piwik\Concurrency\LockBackend\MySqlLockBackend::class),
221	+ );

app/console ADDED Viewed

	@@ -0,0 +1,32 @@


1	+ #!/usr/bin/env php
2	+ <?php
3	+
4	+ use Piwik\FrontController;
5	+
6	+ if (!defined('PIWIK_DOCUMENT_ROOT')) {
7	+ define('PIWIK_DOCUMENT_ROOT', dirname(__FILE__) == '/' ? '' : dirname(__FILE__));
8	+ }
9	+
10	+ if (file_exists(PIWIK_DOCUMENT_ROOT . '/bootstrap.php')) {
11	+ require_once PIWIK_DOCUMENT_ROOT . '/bootstrap.php';
12	+ }
13	+
14	+ if (!defined('PIWIK_INCLUDE_PATH')) {
15	+ define('PIWIK_INCLUDE_PATH', PIWIK_DOCUMENT_ROOT);
16	+ }
17	+
18	+ require_once PIWIK_INCLUDE_PATH . '/core/bootstrap.php';
19	+
20	+ if (!Piwik\Common::isPhpCliMode()) {
21	+ exit;
22	+ }
23	+
24	+ if (!defined('PIWIK_ENABLE_ERROR_HANDLER') \|\| PIWIK_ENABLE_ERROR_HANDLER) {
25	+ Piwik\ErrorHandler::registerErrorHandler();
26	+ Piwik\ExceptionHandler::setUp();
27	+ }
28	+
29	+ FrontController::setUpSafeMode();
30	+
31	+ $console = new Piwik\Console();
32	+ $console->run();

app/core/.htaccess ADDED Viewed

	@@ -0,0 +1,24 @@


1	+ # This file is auto generated by Matomo, do not edit directly
2	+ # Please report any issue or improvement directly to the Matomo team.
3	+
4	+ # First, deny access to all files in this directory
5	+ <Files "*">
6	+ <IfModule mod_version.c>
7	+ <IfVersion < 2.4>
8	+ Order Deny,Allow
9	+ Deny from All
10	+ </IfVersion>
11	+ <IfVersion >= 2.4>
12	+ Require all denied
13	+ </IfVersion>
14	+ </IfModule>
15	+ <IfModule !mod_version.c>
16	+ <IfModule !mod_authz_core.c>
17	+ Order Deny,Allow
18	+ Deny from All
19	+ </IfModule>
20	+ <IfModule mod_authz_core.c>
21	+ Require all denied
22	+ </IfModule>
23	+ </IfModule>
24	+ </Files>

app/core/API/ApiRenderer.php ADDED Viewed

	@@ -0,0 +1,152 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik\API;
10	+
11	+ use Exception;
12	+ use Piwik\Common;
13	+ use Piwik\DataTable\Renderer;
14	+ use Piwik\DataTable;
15	+ use Piwik\Piwik;
16	+ use Piwik\Plugin;
17	+ use Piwik\SettingsServer;
18	+
19	+ /**
20	+ * API renderer
21	+ */
22	+ abstract class ApiRenderer
23	+ {
24	+ protected $request;
25	+
26	+ protected $hideIdSubDataTable;
27	+
28	+ final public function __construct($request)
29	+ {
30	+ $this->request = $request;
31	+ $this->init();
32	+ }
33	+
34	+ protected function init()
35	+ {
36	+ $this->hideIdSubDataTable = Common::getRequestVar('hideIdSubDatable', false, 'int', $this->request);
37	+ }
38	+
39	+ protected function shouldSendBacktrace()
40	+ {
41	+ return Common::isPhpCliMode() && SettingsServer::isArchivePhpTriggered();
42	+ }
43	+
44	+ abstract public function sendHeader();
45	+
46	+ public function renderSuccess($message)
47	+ {
48	+ return 'Success:' . $message;
49	+ }
50	+
51	+ /**
52	+ * @param $message
53	+ * @param Exception\|\Throwable $exception
54	+ * @return mixed
55	+ */
56	+ public function renderException($message, $exception)
57	+ {
58	+ return $message;
59	+ }
60	+
61	+ public function renderScalar($scalar)
62	+ {
63	+ $dataTable = new DataTable\Simple();
64	+ $dataTable->addRowsFromArray(array($scalar));
65	+ return $this->renderDataTable($dataTable);
66	+ }
67	+
68	+ public function renderDataTable($dataTable)
69	+ {
70	+ $renderer = $this->buildDataTableRenderer($dataTable);
71	+ return $renderer->render();
72	+ }
73	+
74	+ public function renderArray($array)
75	+ {
76	+ $renderer = $this->buildDataTableRenderer($array);
77	+ return $renderer->render();
78	+ }
79	+
80	+ public function renderObject($object)
81	+ {
82	+ $exception = new Exception('The API cannot handle this data structure.');
83	+ return $this->renderException($exception->getMessage(), $exception);
84	+ }
85	+
86	+ public function renderResource($resource)
87	+ {
88	+ $exception = new Exception('The API cannot handle this data structure.');
89	+ return $this->renderException($exception->getMessage(), $exception);
90	+ }
91	+
92	+ /**
93	+ * @param $dataTable
94	+ * @return Renderer
95	+ */
96	+ protected function buildDataTableRenderer($dataTable)
97	+ {
98	+ $format = self::getFormatFromClass(get_class($this));
99	+ if ($format == 'json2') {
100	+ $format = 'json';
101	+ }
102	+
103	+ $idSite = Common::getRequestVar('idSite', 0, 'int', $this->request);
104	+
105	+ if (empty($idSite)) {
106	+ $idSite = 'all';
107	+ }
108	+
109	+ $renderer = Renderer::factory($format);
110	+ $renderer->setTable($dataTable);
111	+ $renderer->setIdSite($idSite);
112	+ $renderer->setRenderSubTables(Common::getRequestVar('expanded', false, 'int', $this->request));
113	+ $renderer->setHideIdSubDatableFromResponse($this->hideIdSubDataTable);
114	+
115	+ return $renderer;
116	+ }
117	+
118	+ /**
119	+ * @param string $format
120	+ * @param array $request
121	+ * @return ApiRenderer
122	+ * @throws Exception
123	+ */
124	+ public static function factory($format, $request)
125	+ {
126	+ $formatToCheck = '\\' . ucfirst(strtolower($format));
127	+
128	+ $rendererClassnames = Plugin\Manager::getInstance()->findMultipleComponents('Renderer', 'Piwik\\API\\ApiRenderer');
129	+
130	+ foreach ($rendererClassnames as $klassName) {
131	+ if (Common::stringEndsWith($klassName, $formatToCheck)) {
132	+ return new $klassName($request);
133	+ }
134	+ }
135	+
136	+ $availableRenderers = array();
137	+ foreach ($rendererClassnames as $rendererClassname) {
138	+ $availableRenderers[] = self::getFormatFromClass($rendererClassname);
139	+ }
140	+
141	+ $availableRenderers = implode(', ', $availableRenderers);
142	+ Common::sendHeader('Content-Type: text/plain; charset=utf-8');
143	+ throw new Exception(Piwik::translate('General_ExceptionInvalidRendererFormat', array($format, $availableRenderers)));
144	+ }
145	+
146	+ private static function getFormatFromClass($klassname)
147	+ {
148	+ $klass = explode('\\', $klassname);
149	+
150	+ return strtolower(end($klass));
151	+ }
152	+ }

app/core/API/CORSHandler.php ADDED Viewed

	@@ -0,0 +1,57 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik\API;
10	+
11	+ use Piwik\Common;
12	+ use Piwik\Url;
13	+
14	+ class CORSHandler
15	+ {
16	+ /**
17	+ * @var array
18	+ */
19	+ protected $domains;
20	+
21	+ public function __construct()
22	+ {
23	+ $this->domains = Url::getCorsHostsFromConfig();
24	+ }
25	+
26	+ public function handle()
27	+ {
28	+ if (empty($this->domains)) {
29	+ return;
30	+ }
31	+
32	+ Common::sendHeader('Vary: Origin');
33	+
34	+ // allow Piwik to serve data to all domains
35	+ if (in_array("*", $this->domains)) {
36	+
37	+ Common::sendHeader('Access-Control-Allow-Credentials: true');
38	+
39	+ if (!empty($_SERVER['HTTP_ORIGIN'])) {
40	+ Common::sendHeader('Access-Control-Allow-Origin: ' . $_SERVER['HTTP_ORIGIN']);
41	+ return;
42	+ }
43	+
44	+ Common::sendHeader('Access-Control-Allow-Origin: *');
45	+ return;
46	+ }
47	+
48	+ // specifically allow if it is one of the whitelisted CORS domains
49	+ if (!empty($_SERVER['HTTP_ORIGIN'])) {
50	+ $origin = $_SERVER['HTTP_ORIGIN'];
51	+ if (in_array($origin, $this->domains, true)) {
52	+ Common::sendHeader('Access-Control-Allow-Credentials: true');
53	+ Common::sendHeader('Access-Control-Allow-Origin: ' . $_SERVER['HTTP_ORIGIN']);
54	+ }
55	+ }
56	+ }
57	+ }

app/core/API/DataTableGenericFilter.php ADDED Viewed

	@@ -0,0 +1,245 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik\API;
10	+
11	+ use Exception;
12	+ use Piwik\Common;
13	+ use Piwik\DataTable;
14	+ use Piwik\Plugin\ProcessedMetric;
15	+ use Piwik\Plugin\Report;
16	+
17	+ class DataTableGenericFilter
18	+ {
19	+ /**
20	+ * List of filter names not to run.
21	+ *
22	+ * @var string[]
23	+ */
24	+ private $disabledFilters = array();
25	+
26	+ /**
27	+ * @var Report
28	+ */
29	+ private $report;
30	+
31	+ /**
32	+ * @var array
33	+ */
34	+ private $request;
35	+
36	+ /**
37	+ * Constructor
38	+ *
39	+ * @param $request
40	+ */
41	+ public function __construct($request, $report)
42	+ {
43	+ $this->request = $request;
44	+ $this->report = $report;
45	+ }
46	+
47	+ /**
48	+ * Filters the given data table
49	+ *
50	+ * @param DataTable $table
51	+ */
52	+ public function filter($table)
53	+ {
54	+ $this->applyGenericFilters($table);
55	+ }
56	+
57	+ /**
58	+ * Makes sure a set of filters are not run.
59	+ *
60	+ * @param string[] $filterNames The name of each filter to disable.
61	+ */
62	+ public function disableFilters($filterNames)
63	+ {
64	+ $this->disabledFilters = array_unique(array_merge($this->disabledFilters, $filterNames));
65	+ }
66	+
67	+ /**
68	+ * Returns an array containing the information of the generic Filter
69	+ * to be applied automatically to the data resulting from the API calls.
70	+ *
71	+ * Order to apply the filters:
72	+ * 1 - Filter that remove filtered rows
73	+ * 2 - Filter that sort the remaining rows
74	+ * 3 - Filter that keep only a subset of the results
75	+ * 4 - Presentation filters
76	+ *
77	+ * @return array See the code for spec
78	+ */
79	+ public static function getGenericFiltersInformation()
80	+ {
81	+ return array(
82	+ array('Pattern',
83	+ array(
84	+ 'filter_column' => array('string', 'label'),
85	+ 'filter_pattern' => array('string')
86	+ )),
87	+ array('PatternRecursive',
88	+ array(
89	+ 'filter_column_recursive' => array('string', 'label'),
90	+ 'filter_pattern_recursive' => array('string'),
91	+ )),
92	+ array('ExcludeLowPopulation',
93	+ array(
94	+ 'filter_excludelowpop' => array('string'),
95	+ 'filter_excludelowpop_value' => array('float', '0'),
96	+ )),
97	+ array('Sort',
98	+ array(
99	+ 'filter_sort_column' => array('string'),
100	+ 'filter_sort_order' => array('string', 'desc'),
101	+ $naturalSort = true,
102	+ $recursiveSort = true,
103	+ 'filter_sort_column_secondary' => true
104	+ )),
105	+ array('Truncate',
106	+ array(
107	+ 'filter_truncate' => array('integer'),
108	+ )),
109	+ array('Limit',
110	+ array(
111	+ 'filter_offset' => array('integer', '0'),
112	+ 'filter_limit' => array('integer'),
113	+ 'keep_summary_row' => array('integer', '0'),
114	+ ))
115	+ );
116	+ }
117	+
118	+ private function getGenericFiltersHavingDefaultValues()
119	+ {
120	+ $filters = self::getGenericFiltersInformation();
121	+
122	+ if ($this->report && $this->report->getDefaultSortColumn()) {
123	+ foreach ($filters as $index => $filter) {
124	+ if ($filter[0] === 'Sort') {
125	+ $filters[$index][1]['filter_sort_column'] = array('string', $this->report->getDefaultSortColumn());
126	+ $filters[$index][1]['filter_sort_order'] = array('string', $this->report->getDefaultSortOrder());
127	+
128	+ $callback = $this->report->getSecondarySortColumnCallback();
129	+
130	+ if (is_callable($callback)) {
131	+ $filters[$index][1]['filter_sort_column_secondary'] = $callback;
132	+ }
133	+
134	+ }
135	+ }
136	+ }
137	+
138	+ return $filters;
139	+ }
140	+
141	+ /**
142	+ * Apply generic filters to the DataTable object resulting from the API Call.
143	+ * Disable this feature by setting the parameter disable_generic_filters to 1 in the API call request.
144	+ *
145	+ * @param DataTable $datatable
146	+ * @return bool
147	+ */
148	+ protected function applyGenericFilters($datatable)
149	+ {
150	+ if ($datatable instanceof DataTable\Map) {
151	+ $tables = $datatable->getDataTables();
152	+ foreach ($tables as $table) {
153	+ $this->applyGenericFilters($table);
154	+ }
155	+ return;
156	+ }
157	+
158	+ $tableDisabledFilters = $datatable->getMetadata(DataTable::GENERIC_FILTERS_TO_DISABLE_METADATA_NAME) ?: [];
159	+ $genericFilters = $this->getGenericFiltersHavingDefaultValues();
160	+
161	+ $filterApplied = false;
162	+ foreach ($genericFilters as $filterMeta) {
163	+ $filterName = $filterMeta[0];
164	+ $filterParams = $filterMeta[1];
165	+ $filterParameters = array();
166	+ $exceptionRaised = false;
167	+
168	+ if (in_array($filterName, $this->disabledFilters)
169	+ \|\| in_array($filterName, $tableDisabledFilters)
170	+ ) {
171	+ continue;
172	+ }
173	+
174	+ foreach ($filterParams as $name => $info) {
175	+ if (!is_array($info)) {
176	+ // hard coded value that cannot be changed via API, see eg $naturalSort = true in 'Sort'
177	+ $filterParameters[] = $info;
178	+ } else {
179	+ // parameter type to cast to
180	+ $type = $info[0];
181	+
182	+ // default value if specified, when the parameter doesn't have a value
183	+ $defaultValue = null;
184	+ if (isset($info[1])) {
185	+ $defaultValue = $info[1];
186	+ }
187	+
188	+ try {
189	+ $value = Common::getRequestVar($name, $defaultValue, $type, $this->request);
190	+ settype($value, $type);
191	+ $filterParameters[] = $value;
192	+ } catch (Exception $e) {
193	+ $exceptionRaised = true;
194	+ break;
195	+ }
196	+ }
197	+ }
198	+
199	+ if (!$exceptionRaised) {
200	+ $datatable->filter($filterName, $filterParameters);
201	+ $filterApplied = true;
202	+ }
203	+ }
204	+
205	+ return $filterApplied;
206	+ }
207	+
208	+ public function areProcessedMetricsNeededFor($metrics)
209	+ {
210	+ $columnQueryParameters = array(
211	+ 'filter_column',
212	+ 'filter_column_recursive',
213	+ 'filter_excludelowpop',
214	+ 'filter_sort_column'
215	+ );
216	+
217	+ foreach ($columnQueryParameters as $queryParamName) {
218	+ $queryParamValue = Common::getRequestVar($queryParamName, false, $type = null, $this->request);
219	+ if (!empty($queryParamValue)
220	+ && $this->containsProcessedMetric($metrics, $queryParamValue)
221	+ ) {
222	+ return true;
223	+ }
224	+ }
225	+
226	+ return false;
227	+ }
228	+
229	+ /**
230	+ * @param ProcessedMetric[] $metrics
231	+ * @param string $name
232	+ * @return bool
233	+ */
234	+ private function containsProcessedMetric($metrics, $name)
235	+ {
236	+ foreach ($metrics as $metric) {
237	+ if ($metric instanceof ProcessedMetric
238	+ && $metric->getName() == $name
239	+ ) {
240	+ return true;
241	+ }
242	+ }
243	+ return false;
244	+ }
245	+ }

app/core/API/DataTableManipulator.php ADDED Viewed

	@@ -0,0 +1,208 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik\API;
10	+
11	+ use Exception;
12	+ use Piwik\Archive\DataTableFactory;
13	+ use Piwik\Container\StaticContainer;
14	+ use Piwik\DataTable\Row;
15	+ use Piwik\DataTable;
16	+ use Piwik\Period\Range;
17	+ use Piwik\Plugins\API\API;
18	+
19	+ /**
20	+ * Base class for manipulating data tables.
21	+ * It provides generic mechanisms like iteration and loading subtables.
22	+ *
23	+ * The manipulators are used in ResponseBuilder and are triggered by
24	+ * API parameters. They are not filters because they don't work on the pre-
25	+ * fetched nested data tables. Instead, they load subtables using this base
26	+ * class. This way, they can only load the tables they really need instead
27	+ * of using expanded=1. Another difference between manipulators and filters
28	+ * is that filters keep the overall structure of the table intact while
29	+ * manipulators can change the entire thing.
30	+ */
31	+ abstract class DataTableManipulator
32	+ {
33	+ protected $apiModule;
34	+ protected $apiMethod;
35	+ protected $request;
36	+ protected $apiMethodForSubtable;
37	+
38	+ /**
39	+ * Constructor
40	+ *
41	+ * @param bool $apiModule
42	+ * @param bool $apiMethod
43	+ * @param array $request
44	+ */
45	+ public function __construct($apiModule = false, $apiMethod = false, $request = array())
46	+ {
47	+ $this->apiModule = $apiModule;
48	+ $this->apiMethod = $apiMethod;
49	+ $this->request = $request;
50	+ }
51	+
52	+ /**
53	+ * This method can be used by subclasses to iterate over data tables that might be
54	+ * data table maps. It calls back the template method self::doManipulate for each table.
55	+ * This way, data table arrays can be handled in a transparent fashion.
56	+ *
57	+ * @param DataTable\Map\|DataTable $dataTable
58	+ * @throws Exception
59	+ * @return DataTable\Map\|DataTable
60	+ */
61	+ protected function manipulate($dataTable)
62	+ {
63	+ if ($dataTable instanceof DataTable\Map) {
64	+ return $this->manipulateDataTableMap($dataTable);
65	+ } elseif ($dataTable instanceof DataTable) {
66	+ return $this->manipulateDataTable($dataTable);
67	+ } else {
68	+ return $dataTable;
69	+ }
70	+ }
71	+
72	+ /**
73	+ * Manipulates child DataTables of a DataTable\Map. See @manipulate for more info.
74	+ *
75	+ * @param DataTable\Map $dataTable
76	+ * @return DataTable\Map
77	+ */
78	+ protected function manipulateDataTableMap($dataTable)
79	+ {
80	+ $result = $dataTable->getEmptyClone();
81	+ foreach ($dataTable->getDataTables() as $tableLabel => $childTable) {
82	+ $newTable = $this->manipulate($childTable);
83	+ $result->addTable($newTable, $tableLabel);
84	+ }
85	+ return $result;
86	+ }
87	+
88	+ /**
89	+ * Manipulates a single DataTable instance. Derived classes must define
90	+ * this function.
91	+ */
92	+ abstract protected function manipulateDataTable($dataTable);
93	+
94	+ /**
95	+ * Load the subtable for a row.
96	+ * Returns null if none is found.
97	+ *
98	+ * @param DataTable $dataTable
99	+ * @param Row $row
100	+ *
101	+ * @return DataTable
102	+ */
103	+ protected function loadSubtable($dataTable, $row)
104	+ {
105	+ if (!($this->apiModule && $this->apiMethod && count($this->request))) {
106	+ return null;
107	+ }
108	+
109	+ $request = $this->request;
110	+
111	+ $idSubTable = $row->getIdSubDataTable();
112	+ if ($idSubTable === null) {
113	+ return null;
114	+ }
115	+
116	+ $request['idSubtable'] = $idSubTable;
117	+ if ($dataTable) {
118	+ $period = $dataTable->getMetadata(DataTableFactory::TABLE_METADATA_PERIOD_INDEX);
119	+ if ($period instanceof Range) {
120	+ $request['date'] = $period->getDateStart() . ',' . $period->getDateEnd();
121	+ } else {
122	+ $request['date'] = $period->getDateStart()->toString();
123	+ }
124	+ }
125	+
126	+ $method = $this->getApiMethodForSubtable($request);
127	+ return $this->callApiAndReturnDataTable($this->apiModule, $method, $request);
128	+ }
129	+
130	+ /**
131	+ * In this method, subclasses can clean up the request array for loading subtables
132	+ * in order to make ResponseBuilder behave correctly (e.g. not trigger the
133	+ * manipulator again).
134	+ *
135	+ * @param $request
136	+ * @return
137	+ */
138	+ abstract protected function manipulateSubtableRequest($request);
139	+
140	+ /**
141	+ * Extract the API method for loading subtables from the meta data
142	+ *
143	+ * @throws Exception
144	+ * @return string
145	+ */
146	+ protected function getApiMethodForSubtable($request)
147	+ {
148	+ if (!$this->apiMethodForSubtable) {
149	+ if (!empty($request['idSite'])) {
150	+ $idSite = $request['idSite'];
151	+ } else {
152	+ $idSite = 'all';
153	+ }
154	+
155	+ $apiParameters = array();
156	+ $entityNames = StaticContainer::get('entities.idNames');
157	+ foreach ($entityNames as $idName) {
158	+ if (!empty($request[$idName])) {
159	+ $apiParameters[$idName] = $request[$idName];
160	+ }
161	+ }
162	+
163	+ $meta = API::getInstance()->getMetadata($idSite, $this->apiModule, $this->apiMethod, $apiParameters);
164	+
165	+ if (empty($meta) && array_key_exists('idGoal', $apiParameters)) {
166	+ unset($apiParameters['idGoal']);
167	+ $meta = API::getInstance()->getMetadata($idSite, $this->apiModule, $this->apiMethod, $apiParameters);
168	+ }
169	+
170	+ if (empty($meta)) {
171	+ throw new Exception(sprintf(
172	+ "The DataTable cannot be manipulated: Metadata for report %s.%s could not be found. You can define the metadata in a hook, see example at: https://developer.matomo.org/api-reference/events#apigetreportmetadata",
173	+ $this->apiModule, $this->apiMethod
174	+ ));
175	+ }
176	+
177	+ if (isset($meta[0]['actionToLoadSubTables'])) {
178	+ $this->apiMethodForSubtable = $meta[0]['actionToLoadSubTables'];
179	+ } else {
180	+ $this->apiMethodForSubtable = $this->apiMethod;
181	+ }
182	+ }
183	+ return $this->apiMethodForSubtable;
184	+ }
185	+
186	+ protected function callApiAndReturnDataTable($apiModule, $method, $request)
187	+ {
188	+ $class = Request::getClassNameAPI($apiModule);
189	+
190	+ $request = $this->manipulateSubtableRequest($request);
191	+ $request['serialize'] = 0;
192	+ $request['expanded'] = 0;
193	+ $request['format'] = 'original';
194	+ $request['format_metrics'] = 0;
195	+ $request['compare'] = 0;
196	+
197	+ // don't want to run recursive filters on the subtables as they are loaded,
198	+ // otherwise the result will be empty in places (or everywhere). instead we
199	+ // run it on the flattened table.
200	+ unset($request['filter_pattern_recursive']);
201	+
202	+ $dataTable = Proxy::getInstance()->call($class, $method, $request);
203	+ $response = new ResponseBuilder($format = 'original', $request);
204	+ $response->disableSendHeader();
205	+ $dataTable = $response->getResponse($dataTable, $apiModule, $method);
206	+ return $dataTable;
207	+ }
208	+ }

app/core/API/DataTableManipulator/Flattener.php ADDED Viewed

	@@ -0,0 +1,236 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik\API\DataTableManipulator;
10	+
11	+ use Piwik\API\DataTableManipulator;
12	+ use Piwik\Common;
13	+ use Piwik\DataTable;
14	+ use Piwik\DataTable\Row;
15	+ use Piwik\Plugin\ReportsProvider;
16	+
17	+ /**
18	+ * This class is responsible for flattening data tables.
19	+ *
20	+ * It loads subtables and combines them into a single table by concatenating the labels.
21	+ * This manipulator is triggered by using flat=1 in the API request.
22	+ */
23	+ class Flattener extends DataTableManipulator
24	+ {
25	+
26	+ private $includeAggregateRows = false;
27	+
28	+ /**
29	+ * If the flattener is used after calling this method, aggregate rows will
30	+ * be included in the result. This can be useful when they contain data that
31	+ * the leafs don't have (e.g. conversion stats in some cases).
32	+ */
33	+ public function includeAggregateRows()
34	+ {
35	+ $this->includeAggregateRows = true;
36	+ }
37	+
38	+ /**
39	+ * Separator for building recursive labels (or paths)
40	+ * @var string
41	+ */
42	+ public $recursiveLabelSeparator = '';
43	+
44	+ /**
45	+ * @param DataTable $dataTable
46	+ * @param string $recursiveLabelSeparator
47	+ * @return DataTable\|DataTable\Map
48	+ */
49	+ public function flatten($dataTable, $recursiveLabelSeparator)
50	+ {
51	+ $this->recursiveLabelSeparator = $recursiveLabelSeparator;
52	+
53	+ return $this->manipulate($dataTable);
54	+ }
55	+
56	+ /**
57	+ * Template method called from self::manipulate.
58	+ * Flatten each data table.
59	+ *
60	+ * @param DataTable $dataTable
61	+ * @return DataTable
62	+ */
63	+ protected function manipulateDataTable($dataTable)
64	+ {
65	+ $newDataTable = $dataTable->getEmptyClone($keepFilters = true);
66	+ if ($dataTable->getTotalsRow()) {
67	+ $newDataTable->setTotalsRow($dataTable->getTotalsRow());
68	+ }
69	+
70	+ // this recursive filter will be applied to subtables
71	+ $dataTable->filter('ReplaceSummaryRowLabel');
72	+ $dataTable->filter('ReplaceColumnNames');
73	+
74	+ $report = ReportsProvider::factory($this->apiModule, $this->apiMethod);
75	+ if (!empty($report)) {
76	+ $dimension = $report->getDimension();
77	+ }
78	+
79	+ $dimensionName = !empty($dimension) ? str_replace('.', '_', $dimension->getId()) : 'label1';
80	+
81	+ $this->flattenDataTableInto($dataTable, $newDataTable, $level = 1, $dimensionName);
82	+
83	+ return $newDataTable;
84	+ }
85	+
86	+ /**
87	+ * @param $dataTable DataTable
88	+ * @param $newDataTable
89	+ * @param $dimensionName
90	+ */
91	+ protected function flattenDataTableInto($dataTable, $newDataTable, $level, $dimensionName, $prefix = '', $logo = false)
92	+ {
93	+ foreach ($dataTable->getRows() as $rowId => $row) {
94	+ $this->flattenRow($row, $rowId, $newDataTable, $level, $dimensionName, $prefix, $logo);
95	+ }
96	+ }
97	+
98	+ /**
99	+ * @param Row $row
100	+ * @param DataTable $dataTable
101	+ * @param string $labelPrefix
102	+ * @param string $dimensionName
103	+ * @param bool $parentLogo
104	+ */
105	+ private function flattenRow(Row $row, $rowId, DataTable $dataTable, $level, $dimensionName,
106	+ $labelPrefix = '', $parentLogo = false)
107	+ {
108	+ $dimensions = $dataTable->getMetadata('dimensions');
109	+
110	+ if (empty($dimensions)) {
111	+ $dimensions = [];
112	+ }
113	+
114	+ if (!in_array($dimensionName, $dimensions)) {
115	+ $dimensions[] = $dimensionName;
116	+ }
117	+
118	+ $dataTable->setMetadata('dimensions', $dimensions);
119	+
120	+ $origLabel = $label = $row->getColumn('label');
121	+
122	+ if ($label !== false) {
123	+ $origLabel = $label = trim($label);
124	+
125	+ if ($this->recursiveLabelSeparator == '/') {
126	+ if (substr($label, 0, 1) == '/' && substr($labelPrefix, -1) == '/') {
127	+ $origLabel = $label = substr($label, 1);
128	+ } elseif ($rowId === DataTable::ID_SUMMARY_ROW && $labelPrefix && $label != DataTable::LABEL_SUMMARY_ROW) {
129	+ $label = ' - ' . $label;
130	+ }
131	+ }
132	+
133	+ if ($rowId === DataTable::ID_SUMMARY_ROW) {
134	+ if ($row->getMetadata('url')) {
135	+ // remove url metadata for flattened summary rows
136	+ $row->deleteMetadata('url');
137	+ }
138	+ $row->setMetadata('is_summary', true);
139	+ }
140	+
141	+ $label = $labelPrefix . $label;
142	+ $row->setColumn('label', $label);
143	+
144	+ if ($row->getMetadata($dimensionName)) {
145	+ if ($rowId === DataTable::ID_SUMMARY_ROW && $this->recursiveLabelSeparator == '/') {
146	+ $origLabel = $row->getMetadata($dimensionName) . $this->recursiveLabelSeparator . ' - ' . $origLabel;
147	+ } else {
148	+ $origLabel = $row->getMetadata($dimensionName) . $this->recursiveLabelSeparator . $origLabel;
149	+ }
150	+ }
151	+
152	+ $row->setMetadata($dimensionName, $origLabel);
153	+ }
154	+
155	+ $logo = $row->getMetadata('logo');
156	+ if ($logo === false && $parentLogo !== false) {
157	+ $logo = $parentLogo;
158	+ $row->setMetadata('logo', $logo);
159	+ }
160	+
161	+ /** @var DataTable $subTable */
162	+ $subTable = $row->getSubtable();
163	+
164	+ if ($subTable) {
165	+ $subTable->applyQueuedFilters();
166	+ $row->deleteMetadata('idsubdatatable_in_db');
167	+ } else {
168	+ $subTable = $this->loadSubtable($dataTable, $row);
169	+ }
170	+
171	+ $row->removeSubtable();
172	+
173	+ if ($subTable === null) {
174	+ if ($this->includeAggregateRows) {
175	+ $row->setMetadata('is_aggregate', 0);
176	+ }
177	+ $dataTable->addRow($row);
178	+ } else {
179	+ if ($this->includeAggregateRows) {
180	+ $row->setMetadata('is_aggregate', 1);
181	+ $dataTable->addRow($row);
182	+ }
183	+ $prefix = $label . $this->recursiveLabelSeparator;
184	+
185	+ $report = ReportsProvider::factory($this->apiModule, $this->apiMethod);
186	+ if (!empty($report)) {
187	+ $subDimension = $report->getSubtableDimension();
188	+ }
189	+
190	+ if ($level === 2) {
191	+ $subDimension = $report->getThirdLeveltableDimension();
192	+ }
193	+
194	+ if (empty($subDimension)) {
195	+ $report = ReportsProvider::factory($this->apiModule, $this->getApiMethodForSubtable($this->request));
196	+ $subDimension = $report->getDimension();
197	+ }
198	+
199	+ $subDimensionName = $subDimension ? str_replace('.', '_', $subDimension->getId()) : 'label' . (substr_count($prefix, $this->recursiveLabelSeparator) + 1);
200	+
201	+ if ($origLabel !== false) {
202	+ foreach ($subTable->getRows() as $subRow) {
203	+ foreach ($row->getMetadata() as $name => $value) {
204	+ // do not set 'segment' parameter if there is a segmentValue on the row, since that will prevent the segmentValue
205	+ // from being used in DataTablePostProcessor
206	+ if ($name == 'segment' && $subRow->getMetadata('segmentValue') !== false) {
207	+ continue;
208	+ }
209	+
210	+ if ($subRow->getMetadata($name) === false) {
211	+ $subRow->setMetadata($name, $value);
212	+ }
213	+ }
214	+
215	+ $subRow->setMetadata($dimensionName, $origLabel);
216	+ }
217	+ }
218	+
219	+ $this->flattenDataTableInto($subTable, $dataTable, $level + 1, $subDimensionName, $prefix, $logo);
220	+ }
221	+ }
222	+
223	+ /**
224	+ * Remove the flat parameter from the subtable request
225	+ *
226	+ * @param array $request
227	+ * @return array
228	+ */
229	+ protected function manipulateSubtableRequest($request)
230	+ {
231	+ unset($request['flat']);
232	+
233	+ return $request;
234	+ }
235	+
236	+ }

app/core/API/DataTableManipulator/LabelFilter.php ADDED Viewed

	@@ -0,0 +1,222 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik\API\DataTableManipulator;
10	+
11	+ use Piwik\API\DataTableManipulator;
12	+ use Piwik\Common;
13	+ use Piwik\DataTable;
14	+ use Piwik\DataTable\Row;
15	+
16	+ /**
17	+ * This class is responsible for handling the label parameter that can be
18	+ * added to every API call. If the parameter is set, only the row with the matching
19	+ * label is returned.
20	+ *
21	+ * The labels passed to this class should be urlencoded.
22	+ * Some reports use recursive labels (e.g. action reports). Use > to join them.
23	+ */
24	+ class LabelFilter extends DataTableManipulator
25	+ {
26	+ const SEPARATOR_RECURSIVE_LABEL = '>';
27	+ const TERMINAL_OPERATOR = '@';
28	+
29	+ private $labels;
30	+ private $addLabelIndex;
31	+ private $isComparing;
32	+ private $labelSeries;
33	+ const FLAG_IS_ROW_EVOLUTION = 'label_index';
34	+
35	+ /**
36	+ * Filter a data table by label.
37	+ * The filtered table is returned, which might be a new instance.
38	+ *
39	+ * $apiModule, $apiMethod and $request are needed load sub-datatables
40	+ * for the recursive search. If the label is not recursive, these parameters
41	+ * are not needed.
42	+ *
43	+ * @param string $labels the labels to search for
44	+ * @param DataTable $dataTable the data table to be filtered
45	+ * @param bool $addLabelIndex Whether to add label_index metadata describing which
46	+ * label a row corresponds to.
47	+ * @return DataTable
48	+ */
49	+ public function filter($labels, $dataTable, $addLabelIndex = false)
50	+ {
51	+ if (!is_array($labels)) {
52	+ $labels = array($labels);
53	+ }
54	+
55	+ $this->labels = array_values($labels);
56	+ $this->addLabelIndex = (bool)$addLabelIndex;
57	+ $this->isComparing = $this->isComparing();
58	+
59	+ $labelSeries = Common::getRequestVar('labelSeries', '', 'string', $this->request);
60	+ $labelSeries = explode(',', $labelSeries);
61	+ $labelSeries = array_filter($labelSeries, 'strlen');
62	+ $this->labelSeries = $labelSeries;
63	+
64	+ $result = $this->manipulate($dataTable);
65	+
66	+ return $result;
67	+ }
68	+
69	+ /**
70	+ * Method for the recursive descend
71	+ *
72	+ * @param array $labelParts
73	+ * @param DataTable $dataTable
74	+ * @return Row\|bool
75	+ */
76	+ private function doFilterRecursiveDescend($labelParts, $dataTable)
77	+ {
78	+ // we need to make sure to rebuild the index as some filters change the label column directly via
79	+ // $row->setColumn('label', '') which would not be noticed in the label index otherwise.
80	+ $dataTable->rebuildIndex();
81	+
82	+ // search for the first part of the tree search
83	+ $labelPart = array_shift($labelParts);
84	+
85	+ $row = false;
86	+ foreach ($this->getLabelVariations($labelPart) as $labelPart) {
87	+ $row = $dataTable->getRowFromLabel($labelPart);
88	+ if ($row !== false) {
89	+ break;
90	+ }
91	+ }
92	+
93	+ if ($row === false) {
94	+ // not found
95	+ return false;
96	+ }
97	+
98	+ // end of tree search reached
99	+ if (count($labelParts) == 0) {
100	+ return $row;
101	+ }
102	+
103	+ $subTable = $this->loadSubtable($dataTable, $row);
104	+ if ($subTable === null) {
105	+ // no more subtables but label parts left => no match found
106	+ return false;
107	+ }
108	+
109	+ return $this->doFilterRecursiveDescend($labelParts, $subTable);
110	+ }
111	+
112	+ /**
113	+ * Clean up request for ResponseBuilder to behave correctly
114	+ *
115	+ * @param $request
116	+ */
117	+ protected function manipulateSubtableRequest($request)
118	+ {
119	+ unset($request['label']);
120	+ unset($request['flat']);
121	+ $request['totals'] = 0;
122	+ $request['filter_sort_column'] = ''; // do not sort, we only want to find a matching column
123	+
124	+ return $request;
125	+ }
126	+
127	+ /**
128	+ * Use variations of the label to make it easier to specify the desired label
129	+ *
130	+ * Note: The HTML Encoded version must be tried first, since in ResponseBuilder the $label is unsanitized
131	+ * via Common::unsanitizeLabelParameter.
132	+ *
133	+ * @param string $originalLabel
134	+ * @return array
135	+ */
136	+ private function getLabelVariations($originalLabel)
137	+ {
138	+ static $pageTitleReports = array('getPageTitles', 'getEntryPageTitles', 'getExitPageTitles');
139	+
140	+ $originalLabel = trim($originalLabel);
141	+
142	+ $isTerminal = substr($originalLabel, 0, 1) == self::TERMINAL_OPERATOR;
143	+ if ($isTerminal) {
144	+ $originalLabel = substr($originalLabel, 1);
145	+ }
146	+
147	+ $variations = array();
148	+ $label = trim(urldecode($originalLabel));
149	+
150	+ $sanitizedLabel = Common::sanitizeInputValue($label);
151	+ $variations[] = $sanitizedLabel;
152	+
153	+ if ($this->apiModule == 'Actions'
154	+ && in_array($this->apiMethod, $pageTitleReports)
155	+ ) {
156	+ if ($isTerminal) {
157	+ array_unshift($variations, ' ' . $sanitizedLabel);
158	+ array_unshift($variations, ' ' . $label);
159	+ } else {
160	+ // special case: the Actions.getPageTitles report prefixes some labels with a blank.
161	+ // the blank might be passed by the user but is removed in Request::getRequestArrayFromString.
162	+ $variations[] = ' ' . $sanitizedLabel;
163	+ $variations[] = ' ' . $label;
164	+ }
165	+ }
166	+ $variations[] = $label;
167	+
168	+ $variations = array_unique($variations);
169	+
170	+ return $variations;
171	+ }
172	+
173	+ /**
174	+ * Filter a DataTable instance. See @filter for more info.
175	+ *
176	+ * @param DataTable\Simple\|DataTable\Map $dataTable
177	+ * @return mixed
178	+ */
179	+ protected function manipulateDataTable($dataTable)
180	+ {
181	+ $result = $dataTable->getEmptyClone();
182	+ foreach ($this->labels as $labelIndex => $label) {
183	+ $row = null;
184	+ foreach ($this->getLabelVariations($label) as $labelVariation) {
185	+ $labelVariation = explode(self::SEPARATOR_RECURSIVE_LABEL, $labelVariation);
186	+
187	+ $row = $this->doFilterRecursiveDescend($labelVariation, $dataTable);
188	+ if ($row) {
189	+ if ($this->isComparing
190	+ && isset($this->labelSeries[$labelIndex])
191	+ ) {
192	+ $comparisons = $row->getComparisons();
193	+ if (!empty($comparisons)) {
194	+ $labelSeriesIndex = $this->labelSeries[$labelIndex];
195	+
196	+ $originalLabel = $row->getColumn('label');
197	+
198	+ $row = $comparisons->getRowFromId($labelSeriesIndex);
199	+
200	+ // add label and make sure it is the first column
201	+ $columns = array_merge(['label' => $originalLabel . ' ' . $row->getMetadata('compareSeriesPretty')], $row->getColumns());
202	+ $row->setColumns($columns);
203	+ }
204	+ }
205	+
206	+ if ($this->addLabelIndex) {
207	+ $row->setMetadata(self::FLAG_IS_ROW_EVOLUTION, $labelIndex);
208	+ }
209	+
210	+ $result->addRow($row);
211	+ break;
212	+ }
213	+ }
214	+ }
215	+ return $result;
216	+ }
217	+
218	+ private function isComparing()
219	+ {
220	+ return Common::getRequestVar('compare', 0, 'int', $this->request) == 1;
221	+ }
222	+ }

app/core/API/DataTableManipulator/ReportTotalsCalculator.php ADDED Viewed

	@@ -0,0 +1,251 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik\API\DataTableManipulator;
10	+
11	+ use Piwik\API\DataTableManipulator;
12	+ use Piwik\API\DataTablePostProcessor;
13	+ use Piwik\Common;
14	+ use Piwik\DataTable;
15	+ use Piwik\Metrics;
16	+ use Piwik\Period;
17	+ use Piwik\Piwik;
18	+ use Piwik\Plugin\Report;
19	+ use Piwik\Plugin\ReportsProvider;
20	+
21	+ /**
22	+ * This class is responsible for setting the metadata property 'totals' on each dataTable if the report
23	+ * has a dimension. 'Totals' means it tries to calculate the total report value for each metric. For each
24	+ * the total number of visits, actions, ... for a given report / dataTable.
25	+ */
26	+ class ReportTotalsCalculator extends DataTableManipulator
27	+ {
28	+ /**
29	+ * @var Report
30	+ */
31	+ private $report;
32	+
33	+ /**
34	+ * Constructor
35	+ *
36	+ * @param bool $apiModule
37	+ * @param bool $apiMethod
38	+ * @param array $request
39	+ * @param Report $report
40	+ */
41	+ public function __construct($apiModule = false, $apiMethod = false, $request = array(), $report = null)
42	+ {
43	+ parent::__construct($apiModule, $apiMethod, $request);
44	+ $this->report = $report;
45	+ }
46	+
47	+ /**
48	+ * @param DataTable $table
49	+ * @return \Piwik\DataTable\|\Piwik\DataTable\Map
50	+ */
51	+ public function calculate($table)
52	+ {
53	+ // apiModule and/or apiMethod is empty for instance in case when flat=1 is called. Basically whenever a
54	+ // datamanipulator calls the API and wants the dataTable in return, see callApiAndReturnDataTable().
55	+ // it is also not set for some settings API request etc.
56	+ if (empty($this->apiModule) \|\| empty($this->apiMethod)) {
57	+ return $table;
58	+ }
59	+
60	+ try {
61	+ return $this->manipulate($table);
62	+ } catch (\Exception $e) {
63	+ // eg. requests with idSubtable may trigger this exception
64	+ // (where idSubtable was removed in
65	+ // ?module=API&method=Events.getNameFromCategoryId&idSubtable=1&secondaryDimension=eventName&format=XML&idSite=1&period=day&date=yesterday&flat=0
66	+ return $table;
67	+ }
68	+ }
69	+
70	+ /**
71	+ * Adds ratio metrics if possible.
72	+ *
73	+ * @param DataTable $dataTable
74	+ * @return DataTable
75	+ */
76	+ protected function manipulateDataTable($dataTable)
77	+ {
78	+ if (!empty($this->report) && !$this->report->getDimension() && !$this->isAllMetricsReport()) {
79	+ // we currently do not calculate the total value for reports having no dimension
80	+ return $dataTable;
81	+ }
82	+
83	+ if (1 != Common::getRequestVar('totals', 1, 'integer', $this->request)) {
84	+ return $dataTable;
85	+ }
86	+
87	+ $firstLevelTable = $this->makeSureToWorkOnFirstLevelDataTable($dataTable);
88	+
89	+ if (!$firstLevelTable->getRowsCount()
90	+ \|\| $dataTable->getTotalsRow()
91	+ \|\| $dataTable->getMetadata('totals')
92	+ ) {
93	+ return $dataTable;
94	+ }
95	+
96	+ // keeping queued filters would not only add various metadata but also break the totals calculator for some reports
97	+ // eg when needed metadata is missing to get site information (multisites.getall) etc
98	+ $clone = $firstLevelTable->getEmptyClone($keepFilters = false);
99	+ foreach ($firstLevelTable->getQueuedFilters() as $queuedFilter) {
100	+ if (is_array($queuedFilter) && 'ReplaceColumnNames' === $queuedFilter['className']) {
101	+ $clone->queueFilter($queuedFilter['className'], $queuedFilter['parameters']);
102	+ }
103	+ }
104	+ $tableMeta = $firstLevelTable->getMetadata(DataTable::COLUMN_AGGREGATION_OPS_METADATA_NAME);
105	+
106	+ /** @var DataTable\Row $totalRow */
107	+ $totalRow = null;
108	+ foreach ($firstLevelTable->getRows() as $row) {
109	+ if (!isset($totalRow)) {
110	+ $columns = $row->getColumns();
111	+ $columns['label'] = DataTable::LABEL_TOTALS_ROW;
112	+ $totalRow = new DataTable\Row(array(DataTable\Row::COLUMNS => $columns));
113	+ } else {
114	+ $totalRow->sumRow($row, $copyMetadata = false, $tableMeta);
115	+ }
116	+ }
117	+ $clone->addRow($totalRow);
118	+
119	+ if ($this->report
120	+ && $this->report->getProcessedMetrics()
121	+ && array_keys($this->report->getProcessedMetrics()) === array('nb_actions_per_visit', 'avg_time_on_site', 'bounce_rate', 'conversion_rate')) {
122	+ // hack for AllColumns table or default processed metrics
123	+ $clone->filter('AddColumnsProcessedMetrics', array($deleteRowsWithNoVisit = false));
124	+ }
125	+
126	+ $processor = new DataTablePostProcessor($this->apiModule, $this->apiMethod, $this->request);
127	+ $processor->applyComputeProcessedMetrics($clone);
128	+ $clone = $processor->applyQueuedFilters($clone);
129	+ $clone = $processor->applyMetricsFormatting($clone);
130	+
131	+ $totalRow = null;
132	+ foreach ($clone->getRows() as $row) {
133	+ /** * @var DataTable\Row $row */
134	+ if ($row->getColumn('label') === DataTable::LABEL_TOTALS_ROW) {
135	+ $totalRow = $row;
136	+ break;
137	+ }
138	+ }
139	+
140	+ if (!isset($totalRow) && $clone->getRowsCount() === 1) {
141	+ // if for some reason the processor renamed the totals row,
142	+ $totalRow = $clone->getFirstRow();
143	+ }
144	+
145	+ if (isset($totalRow)) {
146	+ $totals = $row->getColumns();
147	+ unset($totals['label']);
148	+ $dataTable->setMetadata('totals', $totals);
149	+
150	+ if (1 === Common::getRequestVar('keep_totals_row', 0, 'integer', $this->request)) {
151	+ $totalLabel = Common::getRequestVar('keep_totals_row_label', Piwik::translate('General_Totals'), 'string', $this->request);
152	+
153	+ $row->deleteMetadata(false);
154	+ $row->setColumn('label', $totalLabel);
155	+ $dataTable->setTotalsRow($row);
156	+ }
157	+ }
158	+
159	+ return $dataTable;
160	+ }
161	+
162	+ private function makeSureToWorkOnFirstLevelDataTable($table)
163	+ {
164	+ if (!array_key_exists('idSubtable', $this->request)) {
165	+ return $table;
166	+ }
167	+
168	+ $firstLevelReport = $this->findFirstLevelReport();
169	+
170	+ if (empty($firstLevelReport)) {
171	+ // it is not a subtable report
172	+ $module = $this->apiModule;
173	+ $action = $this->apiMethod;
174	+ } else {
175	+ $module = $firstLevelReport->getModule();
176	+ $action = $firstLevelReport->getAction();
177	+ }
178	+
179	+ $request = $this->request;
180	+ unset($request['idSubtable']); // to make sure we work on first level table
181	+
182	+ /** @var \Piwik\Period $period */
183	+ $period = $table->getMetadata('period');
184	+
185	+ if (!empty($period)) {
186	+ // we want a dataTable, not a dataTable\map
187	+ if (Period::isMultiplePeriod($request['date'], $request['period']) \|\| 'range' == $period->getLabel()) {
188	+ $request['date'] = $period->getRangeString();
189	+ $request['period'] = 'range';
190	+ } else {
191	+ $request['date'] = $period->getDateStart()->toString();
192	+ $request['period'] = $period->getLabel();
193	+ }
194	+ }
195	+
196	+ $table = $this->callApiAndReturnDataTable($module, $action, $request);
197	+
198	+ if ($table instanceof DataTable\Map) {
199	+ $table = $table->mergeChildren();
200	+ }
201	+
202	+ return $table;
203	+ }
204	+
205	+ /**
206	+ * Make sure to get all rows of the first level table.
207	+ *
208	+ * @param array $request
209	+ * @return array
210	+ */
211	+ protected function manipulateSubtableRequest($request)
212	+ {
213	+ $request['totals'] = 0;
214	+ $request['expanded'] = 0;
215	+ $request['filter_limit'] = -1;
216	+ $request['filter_offset'] = 0;
217	+ $request['filter_sort_column'] = '';
218	+
219	+ $parametersToRemove = array('flat');
220	+
221	+ if (!array_key_exists('idSubtable', $this->request)) {
222	+ $parametersToRemove[] = 'idSubtable';
223	+ }
224	+
225	+ foreach ($parametersToRemove as $param) {
226	+ if (array_key_exists($param, $request)) {
227	+ unset($request[$param]);
228	+ }
229	+ }
230	+ return $request;
231	+ }
232	+
233	+ private function findFirstLevelReport()
234	+ {
235	+ $reports = new ReportsProvider();
236	+ foreach ($reports->getAllReports() as $report) {
237	+ $actionToLoadSubtables = $report->getActionToLoadSubTables();
238	+ if ($actionToLoadSubtables == $this->apiMethod
239	+ && $this->apiModule == $report->getModule()
240	+ ) {
241	+ return $report;
242	+ }
243	+ }
244	+ return null;
245	+ }
246	+
247	+ private function isAllMetricsReport()
248	+ {
249	+ return $this->report->getModule() == 'API' && $this->report->getAction() == 'get';
250	+ }
251	+ }

app/core/API/DataTablePostProcessor.php ADDED Viewed

	@@ -0,0 +1,502 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ */
8	+
9	+ namespace Piwik\API;
10	+
11	+ use Exception;
12	+ use Piwik\API\DataTableManipulator\Flattener;
13	+ use Piwik\API\DataTableManipulator\LabelFilter;
14	+ use Piwik\API\DataTableManipulator\ReportTotalsCalculator;
15	+ use Piwik\Common;
16	+ use Piwik\DataTable;
17	+ use Piwik\DataTable\DataTableInterface;
18	+ use Piwik\DataTable\Filter\PivotByDimension;
19	+ use Piwik\Metrics\Formatter;
20	+ use Piwik\Piwik;
21	+ use Piwik\Plugin\ProcessedMetric;
22	+ use Piwik\Plugin\Report;
23	+ use Piwik\Plugin\ReportsProvider;
24	+ use Piwik\Plugins\API\Filter\DataComparisonFilter;
25	+
26	+ /**
27	+ * Processes DataTables that should be served through Piwik's APIs. This processing handles
28	+ * special query parameters and computes processed metrics. It does not included rendering to
29	+ * output formats (eg, 'xml').
30	+ */
31	+ class DataTablePostProcessor
32	+ {
33	+ const PROCESSED_METRICS_COMPUTED_FLAG = 'processed_metrics_computed';
34	+
35	+ /**
36	+ * @var null\|Report
37	+ */
38	+ private $report;
39	+
40	+ /**
41	+ * @var string[]
42	+ */
43	+ private $request;
44	+
45	+ /**
46	+ * @var string
47	+ */
48	+ private $apiModule;
49	+
50	+ /**
51	+ * @var string
52	+ */
53	+ private $apiMethod;
54	+
55	+ /**
56	+ * @var Inconsistencies
57	+ */
58	+ private $apiInconsistencies;
59	+
60	+ /**
61	+ * @var Formatter
62	+ */
63	+ private $formatter;
64	+
65	+ private $callbackBeforeGenericFilters;
66	+ private $callbackAfterGenericFilters;
67	+
68	+ /**
69	+ * Constructor.
70	+ */
71	+ public function __construct($apiModule, $apiMethod, $request)
72	+ {
73	+ $this->apiModule = $apiModule;
74	+ $this->apiMethod = $apiMethod;
75	+ $this->setRequest($request);
76	+
77	+ $this->report = ReportsProvider::factory($apiModule, $apiMethod);
78	+ $this->apiInconsistencies = new Inconsistencies();
79	+ $this->setFormatter(new Formatter());
80	+ }
81	+
82	+ public function setFormatter(Formatter $formatter)
83	+ {
84	+ $this->formatter = $formatter;
85	+ }
86	+
87	+ public function setRequest($request)
88	+ {
89	+ $this->request = $request;
90	+ }
91	+
92	+ public function setCallbackBeforeGenericFilters($callbackBeforeGenericFilters)
93	+ {
94	+ $this->callbackBeforeGenericFilters = $callbackBeforeGenericFilters;
95	+ }
96	+
97	+ public function setCallbackAfterGenericFilters($callbackAfterGenericFilters)
98	+ {
99	+ $this->callbackAfterGenericFilters = $callbackAfterGenericFilters;
100	+ }
101	+
102	+ /**
103	+ * Apply post-processing logic to a DataTable of a report for an API request.
104	+ *
105	+ * @param DataTableInterface $dataTable The data table to process.
106	+ * @return DataTableInterface A new data table.
107	+ */
108	+ public function process(DataTableInterface $dataTable)
109	+ {
110	+ // TODO: when calculating metrics before hand, only calculate for needed metrics, not all. NOTE:
111	+ // this is non-trivial since it will require, eg, to make sure processed metrics aren't added
112	+ // after pivotBy is handled.
113	+ $dataTable = $this->applyPivotByFilter($dataTable);
114	+ $dataTable = $this->applyTotalsCalculator($dataTable);
115	+ $dataTable = $this->applyFlattener($dataTable);
116	+
117	+ if ($this->callbackBeforeGenericFilters) {
118	+ call_user_func($this->callbackBeforeGenericFilters, $dataTable);
119	+ }
120	+
121	+ $dataTable = $this->applyGenericFilters($dataTable);
122	+ $this->applyComputeProcessedMetrics($dataTable);
123	+ $dataTable = $this->applyComparison($dataTable);
124	+
125	+ if ($this->callbackAfterGenericFilters) {
126	+ call_user_func($this->callbackAfterGenericFilters, $dataTable);
127	+ }
128	+
129	+ // we automatically safe decode all datatable labels (against xss)
130	+ $dataTable->queueFilter('SafeDecodeLabel');
131	+
132	+ $dataTable = $this->convertSegmentValueToSegment($dataTable);
133	+ $dataTable = $this->applyQueuedFilters($dataTable);
134	+ $dataTable = $this->applyRequestedColumnDeletion($dataTable);
135	+ $dataTable = $this->applyLabelFilter($dataTable);
136	+ $dataTable = $this->applyMetricsFormatting($dataTable);
137	+ return $dataTable;
138	+ }
139	+
140	+ private function convertSegmentValueToSegment(DataTableInterface $dataTable)
141	+ {
142	+ $dataTable->filter('AddSegmentBySegmentValue', array($this->report));
143	+ $dataTable->filter('ColumnCallbackDeleteMetadata', array('segmentValue'));
144	+
145	+ return $dataTable;
146	+ }
147	+
148	+ /**
149	+ * @param DataTableInterface $dataTable
150	+ * @return DataTableInterface
151	+ */
152	+ public function applyPivotByFilter(DataTableInterface $dataTable)
153	+ {
154	+ $pivotBy = Common::getRequestVar('pivotBy', false, 'string', $this->request);
155	+ if (!empty($pivotBy)) {
156	+ $this->applyComputeProcessedMetrics($dataTable);
157	+ $dataTable = $this->convertSegmentValueToSegment($dataTable);
158	+
159	+ $pivotByColumn = Common::getRequestVar('pivotByColumn', false, 'string', $this->request);
160	+ $pivotByColumnLimit = Common::getRequestVar('pivotByColumnLimit', false, 'int', $this->request);
161	+
162	+ $dataTable->filter('PivotByDimension', array($this->report, $pivotBy, $pivotByColumn, $pivotByColumnLimit,
163	+ PivotByDimension::isSegmentFetchingEnabledInConfig()));
164	+ $dataTable->filter('ColumnCallbackDeleteMetadata', array('segment'));
165	+ }
166	+ return $dataTable;
167	+ }
168	+
169	+ /**
170	+ * @param DataTableInterface $dataTable
171	+ * @return DataTable\|DataTableInterface\|DataTable\Map
172	+ */
173	+ public function applyFlattener($dataTable)
174	+ {
175	+ if (Common::getRequestVar('flat', '0', 'string', $this->request) == '1') {
176	+ // skip flattening if not supported by report and remove subtables only
177	+ if ($this->report && !$this->report->supportsFlatten()) {
178	+ $dataTable->filter('RemoveSubtables');
179	+ return $dataTable;
180	+ }
181	+
182	+ $flattener = new Flattener($this->apiModule, $this->apiMethod, $this->request);
183	+ if (Common::getRequestVar('include_aggregate_rows', '0', 'string', $this->request) == '1') {
184	+ $flattener->includeAggregateRows();
185	+ }
186	+
187	+ $recursiveLabelSeparator = ' - ';
188	+ if ($this->report) {
189	+ $recursiveLabelSeparator = $this->report->getRecursiveLabelSeparator();
190	+ }
191	+
192	+ $dataTable = $flattener->flatten($dataTable, $recursiveLabelSeparator);
193	+ }
194	+ return $dataTable;
195	+ }
196	+
197	+ /**
198	+ * @param DataTableInterface $dataTable
199	+ * @return DataTableInterface
200	+ */
201	+ public function applyTotalsCalculator($dataTable)
202	+ {
203	+ if (1 == Common::getRequestVar('totals', '1', 'integer', $this->request)) {
204	+ $calculator = new ReportTotalsCalculator($this->apiModule, $this->apiMethod, $this->request, $this->report);
205	+ $dataTable = $calculator->calculate($dataTable);
206	+ }
207	+ return $dataTable;
208	+ }
209	+
210	+ /**
211	+ * @param DataTableInterface $dataTable
212	+ * @return DataTableInterface
213	+ */
214	+ public function applyGenericFilters($dataTable)
215	+ {
216	+ // if the flag disable_generic_filters is defined we skip the generic filters
217	+ if (0 == Common::getRequestVar('disable_generic_filters', '0', 'string', $this->request)) {
218	+ $this->applyProcessedMetricsGenericFilters($dataTable);
219	+
220	+ $genericFilter = new DataTableGenericFilter($this->request, $this->report);
221	+
222	+ $self = $this;
223	+ $report = $this->report;
224	+ $dataTable->filter(function (DataTable $table) use ($genericFilter, $report, $self) {
225	+ $processedMetrics = Report::getProcessedMetricsForTable($table, $report);
226	+ if ($genericFilter->areProcessedMetricsNeededFor($processedMetrics)) {
227	+ $self->computeProcessedMetrics($table);
228	+ }
229	+ });
230	+
231	+ $label = self::getLabelFromRequest($this->request);
232	+ if (!empty($label)) {
233	+ $genericFilter->disableFilters(array('Limit', 'Truncate'));
234	+ }
235	+
236	+ $genericFilter->filter($dataTable);
237	+ }
238	+
239	+ return $dataTable;
240	+ }
241	+
242	+ /**
243	+ * @param DataTableInterface $dataTable
244	+ * @return DataTableInterface
245	+ */
246	+ public function applyProcessedMetricsGenericFilters($dataTable)
247	+ {
248	+ $addNormalProcessedMetrics = null;
249	+ try {
250	+ $addNormalProcessedMetrics = Common::getRequestVar(
251	+ 'filter_add_columns_when_show_all_columns', null, 'integer', $this->request);
252	+ } catch (Exception $ex) {
253	+ // ignore
254	+ }
255	+
256	+ if ($addNormalProcessedMetrics !== null) {
257	+ $dataTable->filter('AddColumnsProcessedMetrics', array($addNormalProcessedMetrics));
258	+ }
259	+
260	+ $addGoalProcessedMetrics = null;
261	+ try {
262	+ $addGoalProcessedMetrics = Common::getRequestVar(
263	+ 'filter_update_columns_when_show_all_goals', false, 'string', $this->request);
264	+ if ((int) $addGoalProcessedMetrics === 0
265	+ && $addGoalProcessedMetrics !== '0'
266	+ && $addGoalProcessedMetrics != Piwik::LABEL_ID_GOAL_IS_ECOMMERCE_ORDER
267	+ && $addGoalProcessedMetrics != Piwik::LABEL_ID_GOAL_IS_ECOMMERCE_CART
268	+ ) {
269	+ $addGoalProcessedMetrics = null;
270	+ }
271	+ } catch (Exception $ex) {
272	+ // ignore
273	+ }
274	+
275	+ $goalsToProcess = null;
276	+ try {
277	+ $goalsToProcess = Common::getRequestVar('filter_show_goal_columns_process_goals', null, 'string', $this->request);
278	+ $goalsToProcess = explode(',', $goalsToProcess);
279	+ $goalsToProcess = array_map('trim', $goalsToProcess);
280	+ $goalsToProcess = array_filter($goalsToProcess);
281	+ } catch (Exception $ex) {
282	+ // ignore
283	+ }
284	+
285	+ if ($addGoalProcessedMetrics !== null) {
286	+ $idGoal = Common::getRequestVar(
287	+ 'idGoal', DataTable\Filter\AddColumnsProcessedMetricsGoal::GOALS_OVERVIEW, 'string', $this->request);
288	+
289	+ $dataTable->filter('AddColumnsProcessedMetricsGoal', array($ignore = true, $idGoal, $goalsToProcess));
290	+ }
291	+
292	+ return $dataTable;
293	+ }
294	+
295	+ /**
296	+ * @param DataTableInterface $dataTable
297	+ * @return DataTableInterface
298	+ */
299	+ public function applyQueuedFilters($dataTable)
300	+ {
301	+ // if the flag disable_queued_filters is defined we skip the filters that were queued
302	+ if (Common::getRequestVar('disable_queued_filters', 0, 'int', $this->request) == 0) {
303	+ $dataTable->applyQueuedFilters();
304	+ }
305	+ return $dataTable;
306	+ }
307	+
308	+ /**
309	+ * @param DataTableInterface $dataTable
310	+ * @return DataTableInterface
311	+ */
312	+ public function applyRequestedColumnDeletion($dataTable)
313	+ {
314	+ // use the ColumnDelete filter if hideColumns/showColumns is provided (must be done
315	+ // after queued filters are run so processed metrics can be removed, too)
316	+ $hideColumns = Common::getRequestVar('hideColumns', '', 'string', $this->request);
317	+ $showColumns = Common::getRequestVar('showColumns', '', 'string', $this->request);
318	+ $showRawMetrics = Common::getRequestVar('showRawMetrics', 0, 'int', $this->request);
319	+ if (!empty($hideColumns)
320	+ \|\| !empty($showColumns)
321	+ ) {
322	+ $dataTable->filter('ColumnDelete', array($hideColumns, $showColumns));
323	+ } else if ($showRawMetrics !== 1) {
324	+ $this->removeTemporaryMetrics($dataTable);
325	+ }
326	+
327	+ return $dataTable;
328	+ }
329	+
330	+ /**
331	+ * @param DataTableInterface $dataTable
332	+ */
333	+ public function removeTemporaryMetrics(DataTableInterface $dataTable)
334	+ {
335	+ $allColumns = !empty($this->report) ? $this->report->getAllMetrics() : array();
336	+
337	+ $report = $this->report;
338	+ $dataTable->filter(function (DataTable $table) use ($report, $allColumns) {
339	+ $processedMetrics = Report::getProcessedMetricsForTable($table, $report);
340	+
341	+ $allTemporaryMetrics = array();
342	+ foreach ($processedMetrics as $metric) {
343	+ $allTemporaryMetrics = array_merge($allTemporaryMetrics, $metric->getTemporaryMetrics());
344	+ }
345	+
346	+ if (!empty($allTemporaryMetrics)) {
347	+ $table->filter('ColumnDelete', array($allTemporaryMetrics));
348	+ }
349	+ });
350	+ }
351	+
352	+ /**
353	+ * @param DataTableInterface $dataTable
354	+ * @return DataTableInterface
355	+ */
356	+ public function applyLabelFilter($dataTable)
357	+ {
358	+ $label = self::getLabelFromRequest($this->request);
359	+
360	+ // apply label filter: only return rows matching the label parameter (more than one if more than one label)
361	+ if (!empty($label)) {
362	+ $addLabelIndex = Common::getRequestVar('labelFilterAddLabelIndex', 0, 'int', $this->request) == 1;
363	+
364	+ $filter = new LabelFilter($this->apiModule, $this->apiMethod, $this->request);
365	+ $dataTable = $filter->filter($label, $dataTable, $addLabelIndex);
366	+ }
367	+ return $dataTable;
368	+ }
369	+
370	+ /**
371	+ * @param DataTableInterface $dataTable
372	+ * @return DataTableInterface
373	+ */
374	+ public function applyMetricsFormatting($dataTable)
375	+ {
376	+ $formatMetrics = Common::getRequestVar('format_metrics', 0, 'string', $this->request);
377	+ if ($formatMetrics == '0') {
378	+ return $dataTable;
379	+ }
380	+
381	+ // in Piwik 2.X & below, metrics are not formatted in API responses except for percents.
382	+ // this code implements this inconsistency
383	+ $onlyFormatPercents = $formatMetrics === 'bc';
384	+
385	+ $metricsToFormat = null;
386	+ if ($onlyFormatPercents) {
387	+ $metricsToFormat = $this->apiInconsistencies->getPercentMetricsToFormat();
388	+ }
389	+
390	+ // 'all' is a special value that indicates we should format non-processed metrics that are identified
391	+ // by string, like 'revenue'. this should be removed when all metrics are using the `Metric` class.
392	+ $formatAll = $formatMetrics === 'all';
393	+
394	+ $dataTable->filter(array($this->formatter, 'formatMetrics'), array($this->report, $metricsToFormat, $formatAll));
395	+ return $dataTable;
396	+ }
397	+
398	+ /**
399	+ * Returns the value for the label query parameter which can be either a string
400	+ * (ie, label=...) or array (ie, label[]=...).
401	+ *
402	+ * @param array $request
403	+ * @return array
404	+ */
405	+ public static function getLabelFromRequest($request)
406	+ {
407	+ $label = Common::getRequestVar('label', array(), 'array', $request);
408	+ if (empty($label)) {
409	+ $label = Common::getRequestVar('label', '', 'string', $request);
410	+ if (!empty($label)) {
411	+ $label = array($label);
412	+ }
413	+ }
414	+
415	+ $label = self::unsanitizeLabelParameter($label);
416	+ return $label;
417	+ }
418	+
419	+ public static function unsanitizeLabelParameter($label)
420	+ {
421	+ // this is needed because Proxy uses Common::getRequestVar which in turn
422	+ // uses Common::sanitizeInputValue. This causes the > that separates recursive labels
423	+ // to become > and we need to undo that here.
424	+ $label = str_replace( htmlentities('>', ENT_COMPAT \| ENT_HTML401, 'UTF-8'), '>', $label);
425	+ return $label;
426	+ }
427	+
428	+ public function computeProcessedMetrics(DataTable $dataTable)
429	+ {
430	+ if ($dataTable->getMetadata(self::PROCESSED_METRICS_COMPUTED_FLAG)) {
431	+ return;
432	+ }
433	+
434	+ /** @var ProcessedMetric[] $processedMetrics */
435	+ $processedMetrics = Report::getProcessedMetricsForTable($dataTable, $this->report);
436	+ if (empty($processedMetrics)) {
437	+ return;
438	+ }
439	+
440	+ $dataTable->setMetadata(self::PROCESSED_METRICS_COMPUTED_FLAG, true);
441	+
442	+ foreach ($processedMetrics as $name => $processedMetric) {
443	+ if (!$processedMetric->beforeCompute($this->report, $dataTable)) {
444	+ continue;
445	+ }
446	+
447	+ foreach ($dataTable->getRows() as $row) {
448	+ if ($row->getColumn($name) !== false) { // only compute the metric if it has not been computed already
449	+ continue;
450	+ }
451	+
452	+ $computedValue = $processedMetric->compute($row);
453	+ if ($computedValue !== false) {
454	+ $row->addColumn($name, $computedValue);
455	+ }
456	+ }
457	+ }
458	+
459	+ foreach ($dataTable->getRows() as $row) {
460	+ $subtable = $row->getSubtable();
461	+ if (!empty($subtable)) {
462	+ foreach ($processedMetrics as $name => $processedMetric) {
463	+ $processedMetric->beforeComputeSubtable($row);
464	+ }
465	+
466	+ $this->computeProcessedMetrics($subtable);
467	+
468	+ foreach ($processedMetrics as $name => $processedMetric) {
469	+ $processedMetric->afterComputeSubtable($row);
470	+ }
471	+ }
472	+ }
473	+ }
474	+
475	+ public function applyComputeProcessedMetrics(DataTableInterface $dataTable)
476	+ {
477	+ $dataTable->filter(array($this, 'computeProcessedMetrics'));
478	+ }
479	+
480	+ public function applyComparison(DataTableInterface $dataTable)
481	+ {
482	+ $compare = Common::getRequestVar('compare', '0', 'int', $this->request);
483	+ if ($compare != 1) {
484	+ return $dataTable;
485	+ }
486	+
487	+ $filter = new DataComparisonFilter($this->request, $this->report);
488	+ $filter->compare($dataTable);
489	+
490	+ $dataTable->filter(function (DataTable $table) {
491	+ foreach ($table->getRows() as $row) {
492	+ $comparisons = $row->getComparisons();
493	+ if (!empty($comparisons)) {
494	+ $this->computeProcessedMetrics($comparisons);
495	+ }
496	+ }
497	+ });
498	+
499	+ return $dataTable;
500	+ }
501	+ }
502	+

app/core/API/DocumentationGenerator.php ADDED Viewed

	@@ -0,0 +1,397 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik\API;
10	+
11	+ use Exception;
12	+ use Piwik\Common;
13	+ use Piwik\Container\StaticContainer;
14	+ use Piwik\Piwik;
15	+ use Piwik\Url;
16	+ use ReflectionClass;
17	+
18	+ /**
19	+ * Possible tags to use in APIs
20	+ *
21	+ * @hide -> Won't be shown in list of all APIs but is also not possible to be called via HTTP API
22	+ * @hideForAll Same as @hide
23	+ * @hideExceptForSuperUser Same as @hide but still shown and possible to be called by a user with super user access
24	+ * @internal -> Won't be shown in list of all APIs but is possible to be called via HTTP API
25	+ */
26	+ class DocumentationGenerator
27	+ {
28	+ protected $countPluginsLoaded = 0;
29	+
30	+ /**
31	+ * trigger loading all plugins with an API.php file in the Proxy
32	+ */
33	+ public function __construct()
34	+ {
35	+ $plugins = \Piwik\Plugin\Manager::getInstance()->getLoadedPluginsName();
36	+ foreach ($plugins as $plugin) {
37	+ try {
38	+ $className = Request::getClassNameAPI($plugin);
39	+ Proxy::getInstance()->registerClass($className);
40	+ } catch (Exception $e) {
41	+ }
42	+ }
43	+ }
44	+
45	+ /**
46	+ * Returns a HTML page containing help for all the successfully loaded APIs.
47	+ *
48	+ * @param bool $outputExampleUrls
49	+ * @return string
50	+ */
51	+ public function getApiDocumentationAsString($outputExampleUrls = true)
52	+ {
53	+ list($toc, $str) = $this->generateDocumentation($outputExampleUrls, $prefixUrls = '', $displayTitlesAsAngularDirective = true);
54	+
55	+ return "<div piwik-content-block content-title='Quick access to APIs' id='topApiRef' name='topApiRef'>
56	+ $toc</div>
57	+ $str";
58	+ }
59	+
60	+ /**
61	+ * Used on developer.piwik.org
62	+ *
63	+ * @param bool\|true $outputExampleUrls
64	+ * @param string $prefixUrls
65	+ * @return string
66	+ */
67	+ public function getApiDocumentationAsStringForDeveloperReference($outputExampleUrls = true, $prefixUrls = '')
68	+ {
69	+ list($toc, $str) = $this->generateDocumentation($outputExampleUrls, $prefixUrls, $displayTitlesAsAngularDirective = false);
70	+
71	+ return "<h2 id='topApiRef' name='topApiRef'>Quick access to APIs</h2>
72	+ $toc
73	+ $str";
74	+ }
75	+
76	+ protected function prepareModuleToDisplay($moduleName)
77	+ {
78	+ return "<a href='#$moduleName'>$moduleName</a><br/>";
79	+ }
80	+
81	+ protected function prepareMethodToDisplay($moduleName, $info, $methods, $class, $outputExampleUrls, $prefixUrls, $displayTitlesAsAngularDirective)
82	+ {
83	+ $str = '';
84	+ $str .= "\n<a name='$moduleName' id='$moduleName'></a>";
85	+ if($displayTitlesAsAngularDirective) {
86	+ $str .= "<div piwik-content-block content-title='Module " . $moduleName . "'>";
87	+ } else {
88	+ $str .= "<h2>Module " . $moduleName . "</h2>";
89	+ }
90	+ $info['__documentation'] = $this->checkDocumentation($info['__documentation']);
91	+ $str .= "<div class='apiDescription'> " . $info['__documentation'] . " </div>";
92	+ foreach ($methods as $methodName) {
93	+ if (Proxy::getInstance()->isDeprecatedMethod($class, $methodName)) {
94	+ continue;
95	+ }
96	+
97	+ $params = $this->getParametersString($class, $methodName);
98	+
99	+ $str .= "\n <div class='apiMethod'>- <b>$moduleName.$methodName </b>" . $params . "";
100	+ $str .= '<small>';
101	+ if ($outputExampleUrls) {
102	+ $str .= $this->addExamples($class, $methodName, $prefixUrls);
103	+ }
104	+ $str .= '</small>';
105	+ $str .= "</div>\n";
106	+ }
107	+
108	+ if($displayTitlesAsAngularDirective) {
109	+ $str .= "</div>";
110	+ }
111	+
112	+ return $str;
113	+ }
114	+
115	+ protected function prepareModulesAndMethods($info, $moduleName)
116	+ {
117	+ $toDisplay = array();
118	+
119	+ foreach ($info as $methodName => $infoMethod) {
120	+ if ($methodName == '__documentation') {
121	+ continue;
122	+ }
123	+ $toDisplay[$moduleName][] = $methodName;
124	+ }
125	+
126	+ return $toDisplay;
127	+ }
128	+
129	+ protected function addExamples($class, $methodName, $prefixUrls)
130	+ {
131	+ $token_auth = "&token_auth=" . Piwik::getCurrentUserTokenAuth();
132	+ $parametersToSet = array(
133	+ 'idSite' => Common::getRequestVar('idSite', 1, 'int'),
134	+ 'period' => Common::getRequestVar('period', 'day', 'string'),
135	+ 'date' => Common::getRequestVar('date', 'today', 'string')
136	+ );
137	+ $str = '';
138	+ $str .= "<span class=\"example\">";
139	+ $exampleUrl = $this->getExampleUrl($class, $methodName, $parametersToSet);
140	+ if ($exampleUrl !== false) {
141	+ $lastNUrls = '';
142	+ if (preg_match('/(&period)\|(&date)/', $exampleUrl)) {
143	+ $exampleUrlRss = $prefixUrls . $this->getExampleUrl($class, $methodName, array('date' => 'last10', 'period' => 'day') + $parametersToSet);
144	+ $lastNUrls = ", RSS of the last <a target='_blank' href='$exampleUrlRss&format=rss$token_auth&translateColumnNames=1'>10 days</a>";
145	+ }
146	+ $exampleUrl = $prefixUrls . $exampleUrl;
147	+ $str .= " [ Example in
148	+ <a target='_blank' href='$exampleUrl&format=xml$token_auth'>XML</a>,
149	+ <a target='_blank' href='$exampleUrl&format=JSON$token_auth'>Json</a>,
150	+ <a target='_blank' href='$exampleUrl&format=Tsv$token_auth&translateColumnNames=1'>Tsv (Excel)</a>
151	+ $lastNUrls
152	+ ]";
153	+ } else {
154	+ $str .= " [ No example available ]";
155	+ }
156	+ $str .= "</span>";
157	+ return $str;
158	+ }
159	+
160	+ /**
161	+ * Check if Class contains @hide
162	+ *
163	+ * @param ReflectionClass $rClass instance of ReflectionMethod
164	+ * @return bool
165	+ */
166	+ public function checkIfClassCommentContainsHideAnnotation(ReflectionClass $rClass)
167	+ {
168	+ return false !== strstr($rClass->getDocComment(), '@hide');
169	+ }
170	+
171	+ /**
172	+ * Check if Class contains @internal
173	+ *
174	+ * @param ReflectionClass\|\ReflectionMethod $rClass instance of ReflectionMethod
175	+ * @return bool
176	+ */
177	+ private function checkIfCommentContainsInternalAnnotation($rClass)
178	+ {
179	+ return false !== strstr($rClass->getDocComment(), '@internal');
180	+ }
181	+
182	+ /**
183	+ * Check if documentation contains @hide annotation and deletes it
184	+ *
185	+ * @param $moduleToCheck
186	+ * @return mixed
187	+ */
188	+ public function checkDocumentation($moduleToCheck)
189	+ {
190	+ if (strpos($moduleToCheck, '@hide') == true) {
191	+ $moduleToCheck = str_replace(strtok(strstr($moduleToCheck, '@hide'), "\n"), "", $moduleToCheck);
192	+ }
193	+ return $moduleToCheck;
194	+ }
195	+
196	+ /**
197	+ * Returns a string containing links to examples on how to call a given method on a given API
198	+ * It will export links to XML, CSV, HTML, JSON, PHP, etc.
199	+ * It will not export links for methods such as deleteSite or deleteUser
200	+ *
201	+ * @param string $class the class
202	+ * @param string $methodName the method
203	+ * @param array $parametersToSet parameters to set
204	+ * @return string\|bool when not possible
205	+ */
206	+ public function getExampleUrl($class, $methodName, $parametersToSet = array())
207	+ {
208	+ $knowExampleDefaultParametersValues = array(
209	+ 'access' => 'view',
210	+ 'userLogin' => 'test',
211	+ 'passwordMd5ied' => 'passwordExample',
212	+ 'email' => 'test@example.org',
213	+
214	+ 'languageCode' => 'fr',
215	+ 'url' => 'https://divezone.net/',
216	+ 'pageUrl' => 'https://divezone.net/',
217	+ 'apiModule' => 'UserCountry',
218	+ 'apiAction' => 'getCountry',
219	+ 'lastMinutes' => '30',
220	+ 'abandonedCarts' => '0',
221	+ 'segmentName' => 'pageTitle',
222	+ 'ip' => '194.57.91.215',
223	+ 'idSites' => '1,2',
224	+ 'idAlert' => '1',
225	+ 'seconds' => '3600',
226	+ // 'segmentName' => 'browserCode',
227	+ );
228	+
229	+ foreach ($parametersToSet as $name => $value) {
230	+ $knowExampleDefaultParametersValues[$name] = $value;
231	+ }
232	+
233	+ // no links for these method names
234	+ $doNotPrintExampleForTheseMethods = array(
235	+ //Sites
236	+ 'deleteSite',
237	+ 'addSite',
238	+ 'updateSite',
239	+ 'addSiteAliasUrls',
240	+ //Users
241	+ 'deleteUser',
242	+ 'addUser',
243	+ 'updateUser',
244	+ 'setUserAccess',
245	+ //Goals
246	+ 'addGoal',
247	+ 'updateGoal',
248	+ 'deleteGoal',
249	+ //Marketplace
250	+ 'deleteLicenseKey'
251	+ );
252	+
253	+ if (in_array($methodName, $doNotPrintExampleForTheseMethods)) {
254	+ return false;
255	+ }
256	+
257	+ // we try to give an URL example to call the API
258	+ $aParameters = Proxy::getInstance()->getParametersList($class, $methodName);
259	+ $aParameters['format'] = false;
260	+ $aParameters['hideIdSubDatable'] = false;
261	+ $aParameters['serialize'] = false;
262	+ $aParameters['language'] = false;
263	+ $aParameters['translateColumnNames'] = false;
264	+ $aParameters['label'] = false;
265	+ $aParameters['labelSeries'] = false;
266	+ $aParameters['flat'] = false;
267	+ $aParameters['include_aggregate_rows'] = false;
268	+ $aParameters['filter_offset'] = false;
269	+ $aParameters['filter_limit'] = false;
270	+ $aParameters['filter_sort_column'] = false;
271	+ $aParameters['filter_sort_order'] = false;
272	+ $aParameters['filter_excludelowpop'] = false;
273	+ $aParameters['filter_excludelowpop_value'] = false;
274	+ $aParameters['filter_column_recursive'] = false;
275	+ $aParameters['filter_pattern'] = false;
276	+ $aParameters['filter_pattern_recursive'] = false;
277	+ $aParameters['filter_truncate'] = false;
278	+ $aParameters['hideColumns'] = false;
279	+ $aParameters['showColumns'] = false;
280	+ $aParameters['filter_pattern_recursive'] = false;
281	+ $aParameters['pivotBy'] = false;
282	+ $aParameters['pivotByColumn'] = false;
283	+ $aParameters['pivotByColumnLimit'] = false;
284	+ $aParameters['disable_queued_filters'] = false;
285	+ $aParameters['disable_generic_filters'] = false;
286	+ $aParameters['expanded'] = false;
287	+ $aParameters['idDimenson'] = false;
288	+ $aParameters['format_metrics'] = false;
289	+ $aParameters['compare'] = false;
290	+ $aParameters['compareDates'] = false;
291	+ $aParameters['comparePeriods'] = false;
292	+ $aParameters['compareSegments'] = false;
293	+ $aParameters['comparisonIdSubtables'] = false;
294	+ $aParameters['invert_compare_change_compute'] = false;
295	+
296	+ $entityNames = StaticContainer::get('entities.idNames');
297	+ foreach ($entityNames as $entityName) {
298	+ if (isset($aParameters[$entityName])) {
299	+ continue;
300	+ }
301	+ $aParameters[$entityName] = false;
302	+ }
303	+
304	+ $moduleName = Proxy::getInstance()->getModuleNameFromClassName($class);
305	+ $aParameters = array_merge(array('module' => 'API', 'method' => $moduleName . '.' . $methodName), $aParameters);
306	+
307	+ foreach ($aParameters as $nameVariable => &$defaultValue) {
308	+ if (isset($knowExampleDefaultParametersValues[$nameVariable])) {
309	+ $defaultValue = $knowExampleDefaultParametersValues[$nameVariable];
310	+ } // if there isn't a default value for a given parameter,
311	+ // we need a 'know default value' or we can't generate the link
312	+ elseif ($defaultValue instanceof NoDefaultValue) {
313	+ return false;
314	+ }
315	+ }
316	+ return '?' . Url::getQueryStringFromParameters($aParameters);
317	+ }
318	+
319	+ /**
320	+ * Returns the methods $class.$name parameters (and default value if provided) as a string.
321	+ *
322	+ * @param string $class The class name
323	+ * @param string $name The method name
324	+ * @return string For example "(idSite, period, date = 'today')"
325	+ */
326	+ protected function getParametersString($class, $name)
327	+ {
328	+ $aParameters = Proxy::getInstance()->getParametersList($class, $name);
329	+ $asParameters = array();
330	+ foreach ($aParameters as $nameVariable => $defaultValue) {
331	+ // Do not show API parameters starting with _
332	+ // They are supposed to be used only in internal API calls
333	+ if (strpos($nameVariable, '_') === 0) {
334	+ continue;
335	+ }
336	+ $str = $nameVariable;
337	+ if (!($defaultValue instanceof NoDefaultValue)) {
338	+ if (is_array($defaultValue)) {
339	+ $str .= " = 'Array'";
340	+ } else {
341	+ $str .= " = '$defaultValue'";
342	+ }
343	+ }
344	+ $asParameters[] = $str;
345	+ }
346	+ $sParameters = implode(", ", $asParameters);
347	+ return "($sParameters)";
348	+ }
349	+
350	+ /**
351	+ * @param $outputExampleUrls
352	+ * @param $prefixUrls
353	+ * @param $displayTitlesAsAngularDirective
354	+ * @return array
355	+ */
356	+ protected function generateDocumentation($outputExampleUrls, $prefixUrls, $displayTitlesAsAngularDirective)
357	+ {
358	+ $str = $toc = '';
359	+
360	+ foreach (Proxy::getInstance()->getMetadata() as $class => $info) {
361	+ $moduleName = Proxy::getInstance()->getModuleNameFromClassName($class);
362	+ $rClass = new ReflectionClass($class);
363	+
364	+ if (!Piwik::hasUserSuperUserAccess() && $this->checkIfClassCommentContainsHideAnnotation($rClass)) {
365	+ continue;
366	+ }
367	+
368	+ if ($this->checkIfCommentContainsInternalAnnotation($rClass)) {
369	+ continue;
370	+ }
371	+
372	+ $toDisplay = $this->prepareModulesAndMethods($info, $moduleName);
373	+
374	+ foreach ($toDisplay as $moduleName => $methods) {
375	+ foreach ($methods as $index => $method) {
376	+ if (!method_exists($class, $method)) { // method is handled through API.Request.intercept event
377	+ continue;
378	+ }
379	+
380	+ $reflectionMethod = new \ReflectionMethod($class, $method);
381	+ if ($this->checkIfCommentContainsInternalAnnotation($reflectionMethod)) {
382	+ unset($toDisplay[$moduleName][$index]);
383	+ }
384	+ }
385	+ if (empty($toDisplay[$moduleName])) {
386	+ unset($toDisplay[$moduleName]);
387	+ }
388	+ }
389	+
390	+ foreach ($toDisplay as $moduleName => $methods) {
391	+ $toc .= $this->prepareModuleToDisplay($moduleName);
392	+ $str .= $this->prepareMethodToDisplay($moduleName, $info, $methods, $class, $outputExampleUrls, $prefixUrls, $displayTitlesAsAngularDirective);
393	+ }
394	+ }
395	+ return array($toc, $str);
396	+ }
397	+ }

app/core/API/Inconsistencies.php ADDED Viewed

	@@ -0,0 +1,47 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ */
8	+ namespace Piwik\API;
9	+
10	+ /**
11	+ * Contains logic to replicate inconsistencies in Piwik's API. This class exists
12	+ * to provide a way to clean up existing Piwik code and behavior without breaking
13	+ * backwards compatibility immediately.
14	+ *
15	+ * Code that handles the case when the 'format_metrics' query parameter value is
16	+ * 'bc' should be removed as well. This code is in API\Request and DataTablePostProcessor.
17	+ *
18	+ * Should be removed before releasing Piwik 3.0.
19	+ */
20	+ class Inconsistencies
21	+ {
22	+ /**
23	+ * In Piwik 2.X and below, the "raw" API would format percent values but no others.
24	+ * This method returns the list of percent metrics that were returned from the API
25	+ * formatted so we can maintain BC.
26	+ *
27	+ * Used by DataTablePostProcessor.
28	+ */
29	+ public function getPercentMetricsToFormat()
30	+ {
31	+ return array(
32	+ 'bounce_rate',
33	+ 'conversion_rate',
34	+ 'abandoned_rate',
35	+ 'interaction_rate',
36	+ 'exit_rate',
37	+ 'bounce_rate_returning',
38	+ 'nb_visits_percentage',
39	+ '/.*_evolution/',
40	+ '/goal_.*_conversion_rate/',
41	+ '/step_.*_rate/',
42	+ '/funnel_.*_rate/',
43	+ '/form_.*_rate/',
44	+ '/field_.*_rate/',
45	+ );
46	+ }
47	+ }

app/core/API/Proxy.php ADDED Viewed

	@@ -0,0 +1,582 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+
10	+ namespace Piwik\API;
11	+
12	+ use Exception;
13	+ use Piwik\Common;
14	+ use Piwik\Container\StaticContainer;
15	+ use Piwik\Context;
16	+ use Piwik\Piwik;
17	+ use Piwik\Plugin\Manager;
18	+ use Piwik\Singleton;
19	+ use ReflectionClass;
20	+ use ReflectionMethod;
21	+
22	+ /**
23	+ * Proxy is a singleton that has the knowledge of every method available, their parameters
24	+ * and default values.
25	+ * Proxy receives all the API calls requests via call() and forwards them to the right
26	+ * object, with the parameters in the right order.
27	+ *
28	+ * It will also log the performance of API calls (time spent, parameter values, etc.) if logger available
29	+ */
30	+ class Proxy
31	+ {
32	+ // array of already registered plugins names
33	+ protected $alreadyRegistered = array();
34	+
35	+ protected $metadataArray = array();
36	+ private $hideIgnoredFunctions = true;
37	+
38	+ // when a parameter doesn't have a default value we use this
39	+ private $noDefaultValue;
40	+
41	+ public function __construct()
42	+ {
43	+ $this->noDefaultValue = new NoDefaultValue();
44	+ }
45	+
46	+ public static function getInstance()
47	+ {
48	+ return StaticContainer::get(self::class);
49	+ }
50	+
51	+ /**
52	+ * Returns array containing reflection meta data for all the loaded classes
53	+ * eg. number of parameters, method names, etc.
54	+ *
55	+ * @return array
56	+ */
57	+ public function getMetadata()
58	+ {
59	+ ksort($this->metadataArray);
60	+ return $this->metadataArray;
61	+ }
62	+
63	+ /**
64	+ * Registers the API information of a given module.
65	+ *
66	+ * The module to be registered must be
67	+ * - a singleton (providing a getInstance() method)
68	+ * - the API file must be located in plugins/ModuleName/API.php
69	+ * for example plugins/Referrers/API.php
70	+ *
71	+ * The method will introspect the methods, their parameters, etc.
72	+ *
73	+ * @param string $className ModuleName eg. "API"
74	+ */
75	+ public function registerClass($className)
76	+ {
77	+ if (isset($this->alreadyRegistered[$className])) {
78	+ return;
79	+ }
80	+ $this->includeApiFile($className);
81	+ $this->checkClassIsSingleton($className);
82	+
83	+ $rClass = new ReflectionClass($className);
84	+ if (!$this->shouldHideAPIMethod($rClass->getDocComment())) {
85	+ foreach ($rClass->getMethods() as $method) {
86	+ $this->loadMethodMetadata($className, $method);
87	+ }
88	+
89	+ $this->setDocumentation($rClass, $className);
90	+ $this->alreadyRegistered[$className] = true;
91	+ }
92	+ }
93	+
94	+ /**
95	+ * Will be displayed in the API page
96	+ *
97	+ * @param ReflectionClass $rClass Instance of ReflectionClass
98	+ * @param string $className Name of the class
99	+ */
100	+ private function setDocumentation($rClass, $className)
101	+ {
102	+ // Doc comment
103	+ $doc = $rClass->getDocComment();
104	+ $doc = str_replace(" * " . PHP_EOL, "<br>", $doc);
105	+
106	+ // boldify the first line only if there is more than one line, otherwise too much bold
107	+ if (substr_count($doc, '<br>') > 1) {
108	+ $firstLineBreak = strpos($doc, "<br>");
109	+ $doc = "<div class='apiFirstLine'>" . substr($doc, 0, $firstLineBreak) . "</div>" . substr($doc, $firstLineBreak + strlen("<br>"));
110	+ }
111	+ $doc = preg_replace("/(@package)[a-z _A-Z]*/", "", $doc);
112	+ $doc = preg_replace("/(@method).*/", "", $doc);
113	+ $doc = str_replace(array("\t", "\n", "/*", "/", " * ", " ", " ", "\t", " * @package"), " ", $doc);
114	+
115	+ // replace 'foo' and `bar` and "foobar" with code blocks... much magic
116	+ $doc = preg_replace('/`(.*?)`/', '<code>$1</code>', $doc);
117	+ $this->metadataArray[$className]['__documentation'] = $doc;
118	+ }
119	+
120	+ /**
121	+ * Returns number of classes already loaded
122	+ * @return int
123	+ */
124	+ public function getCountRegisteredClasses()
125	+ {
126	+ return count($this->alreadyRegistered);
127	+ }
128	+
129	+ /**
130	+ * Will execute $className->$methodName($parametersValues)
131	+ * If any error is detected (wrong number of parameters, method not found, class not found, etc.)
132	+ * it will throw an exception
133	+ *
134	+ * It also logs the API calls, with the parameters values, the returned value, the performance, etc.
135	+ * You can enable logging in config/global.ini.php (log_api_call)
136	+ *
137	+ * @param string $className The class name (eg. API)
138	+ * @param string $methodName The method name
139	+ * @param array $parametersRequest The parameters pairs (name=>value)
140	+ *
141	+ * @return mixed\|null
142	+ * @throws Exception\|\Piwik\NoAccessException
143	+ */
144	+ public function call($className, $methodName, $parametersRequest)
145	+ {
146	+ // Temporarily sets the Request array to this API call context
147	+ return Context::executeWithQueryParameters($parametersRequest, function () use ($className, $methodName, $parametersRequest) {
148	+ $returnedValue = null;
149	+
150	+ $this->registerClass($className);
151	+
152	+ // instanciate the object
153	+ $object = $className::getInstance();
154	+
155	+ // check method exists
156	+ $this->checkMethodExists($className, $methodName);
157	+
158	+ // get the list of parameters required by the method
159	+ $parameterNamesDefaultValues = $this->getParametersList($className, $methodName);
160	+
161	+ // load parameters in the right order, etc.
162	+ $finalParameters = $this->getRequestParametersArray($parameterNamesDefaultValues, $parametersRequest);
163	+
164	+ // allow plugins to manipulate the value
165	+ $pluginName = $this->getModuleNameFromClassName($className);
166	+
167	+ $returnedValue = null;
168	+
169	+ /**
170	+ * Triggered before an API request is dispatched.
171	+ *
172	+ * This event can be used to modify the arguments passed to one or more API methods.
173	+ *
174	+ * Example
175	+ *
176	+ * Piwik::addAction('API.Request.dispatch', function (&$parameters, $pluginName, $methodName) {
177	+ * if ($pluginName == 'Actions') {
178	+ * if ($methodName == 'getPageUrls') {
179	+ * // ... do something ...
180	+ * } else {
181	+ * // ... do something else ...
182	+ * }
183	+ * }
184	+ * });
185	+ *
186	+ * @param array &$finalParameters List of parameters that will be passed to the API method.
187	+ * @param string $pluginName The name of the plugin the API method belongs to.
188	+ * @param string $methodName The name of the API method that will be called.
189	+ */
190	+ Piwik::postEvent('API.Request.dispatch', array(&$finalParameters, $pluginName, $methodName));
191	+
192	+ /**
193	+ * Triggered before an API request is dispatched.
194	+ *
195	+ * This event exists for convenience and is triggered directly after the {@hook API.Request.dispatch}
196	+ * event is triggered. It can be used to modify the arguments passed to a single API method.
197	+ *
198	+ * _Note: This is can be accomplished with the {@hook API.Request.dispatch} event as well, however
199	+ * event handlers for that event will have to do more work._
200	+ *
201	+ * Example
202	+ *
203	+ * Piwik::addAction('API.Actions.getPageUrls', function (&$parameters) {
204	+ * // force use of a single website. for some reason.
205	+ * $parameters['idSite'] = 1;
206	+ * });
207	+ *
208	+ * @param array &$finalParameters List of parameters that will be passed to the API method.
209	+ */
210	+ Piwik::postEvent(sprintf('API.%s.%s', $pluginName, $methodName), array(&$finalParameters));
211	+
212	+ /**
213	+ * Triggered before an API request is dispatched.
214	+ *
215	+ * Use this event to intercept an API request and execute your own code instead. If you set
216	+ * `$returnedValue` in a handler for this event, the original API method will not be executed,
217	+ * and the result will be what you set in the event handler.
218	+ *
219	+ * @param mixed &$returnedValue Set this to set the result and preempt normal API invocation.
220	+ * @param array &$finalParameters List of parameters that will be passed to the API method.
221	+ * @param string $pluginName The name of the plugin the API method belongs to.
222	+ * @param string $methodName The name of the API method that will be called.
223	+ * @param array $parametersRequest The query parameters for this request.
224	+ */
225	+ Piwik::postEvent('API.Request.intercept', [&$returnedValue, $finalParameters, $pluginName, $methodName, $parametersRequest]);
226	+
227	+ $apiParametersInCorrectOrder = array();
228	+
229	+ foreach ($parameterNamesDefaultValues as $name => $defaultValue) {
230	+ if (isset($finalParameters[$name]) \|\| array_key_exists($name, $finalParameters)) {
231	+ $apiParametersInCorrectOrder[] = $finalParameters[$name];
232	+ }
233	+ }
234	+
235	+ // call the method if a hook hasn't already set an output variable
236	+ if ($returnedValue === null) {
237	+ $returnedValue = call_user_func_array(array($object, $methodName), $apiParametersInCorrectOrder);
238	+ }
239	+
240	+ $endHookParams = array(
241	+ &$returnedValue,
242	+ array('className' => $className,
243	+ 'module' => $pluginName,
244	+ 'action' => $methodName,
245	+ 'parameters' => $finalParameters)
246	+ );
247	+
248	+ /**
249	+ * Triggered directly after an API request is dispatched.
250	+ *
251	+ * This event exists for convenience and is triggered immediately before the
252	+ * {@hook API.Request.dispatch.end} event. It can be used to modify the output of a single
253	+ * API method.
254	+ *
255	+ * _Note: This can be accomplished with the {@hook API.Request.dispatch.end} event as well,
256	+ * however event handlers for that event will have to do more work._
257	+ *
258	+ * Example
259	+ *
260	+ * // append (0 hits) to the end of row labels whose row has 0 hits
261	+ * Piwik::addAction('API.Actions.getPageUrls', function (&$returnValue, $info)) {
262	+ * $returnValue->filter('ColumnCallbackReplace', 'label', function ($label, $hits) {
263	+ * if ($hits === 0) {
264	+ * return $label . " (0 hits)";
265	+ * } else {
266	+ * return $label;
267	+ * }
268	+ * }, null, array('nb_hits'));
269	+ * }
270	+ *
271	+ * @param mixed &$returnedValue The API method's return value. Can be an object, such as a
272	+ * {@link Piwik\DataTable DataTable} instance.
273	+ * could be a {@link Piwik\DataTable DataTable}.
274	+ * @param array $extraInfo An array holding information regarding the API request. Will
275	+ * contain the following data:
276	+ *
277	+ * - className: The namespace-d class name of the API instance
278	+ * that's being called.
279	+ * - module: The name of the plugin the API request was
280	+ * dispatched to.
281	+ * - action: The name of the API method that was executed.
282	+ * - parameters: The array of parameters passed to the API
283	+ * method.
284	+ */
285	+ Piwik::postEvent(sprintf('API.%s.%s.end', $pluginName, $methodName), $endHookParams);
286	+
287	+ /**
288	+ * Triggered directly after an API request is dispatched.
289	+ *
290	+ * This event can be used to modify the output of any API method.
291	+ *
292	+ * Example
293	+ *
294	+ * // append (0 hits) to the end of row labels whose row has 0 hits for any report that has the 'nb_hits' metric
295	+ * Piwik::addAction('API.Actions.getPageUrls.end', function (&$returnValue, $info)) {
296	+ * // don't process non-DataTable reports and reports that don't have the nb_hits column
297	+ * if (!($returnValue instanceof DataTableInterface)
298	+ * \|\| in_array('nb_hits', $returnValue->getColumns())
299	+ * ) {
300	+ * return;
301	+ * }
302	+ *
303	+ * $returnValue->filter('ColumnCallbackReplace', 'label', function ($label, $hits) {
304	+ * if ($hits === 0) {
305	+ * return $label . " (0 hits)";
306	+ * } else {
307	+ * return $label;
308	+ * }
309	+ * }, null, array('nb_hits'));
310	+ * }
311	+ *
312	+ * @param mixed &$returnedValue The API method's return value. Can be an object, such as a
313	+ * {@link Piwik\DataTable DataTable} instance.
314	+ * @param array $extraInfo An array holding information regarding the API request. Will
315	+ * contain the following data:
316	+ *
317	+ * - className: The namespace-d class name of the API instance
318	+ * that's being called.
319	+ * - module: The name of the plugin the API request was
320	+ * dispatched to.
321	+ * - action: The name of the API method that was executed.
322	+ * - parameters: The array of parameters passed to the API
323	+ * method.
324	+ */
325	+ Piwik::postEvent('API.Request.dispatch.end', $endHookParams);
326	+
327	+ return $returnedValue;
328	+ });
329	+ }
330	+
331	+ /**
332	+ * Returns the parameters names and default values for the method $name
333	+ * of the class $class
334	+ *
335	+ * @param string $class The class name
336	+ * @param string $name The method name
337	+ * @return array Format array(
338	+ * 'testParameter' => null, // no default value
339	+ * 'life' => 42, // default value = 42
340	+ * 'date' => 'yesterday',
341	+ * );
342	+ */
343	+ public function getParametersList($class, $name)
344	+ {
345	+ return $this->metadataArray[$class][$name]['parameters'];
346	+ }
347	+
348	+ /**
349	+ * Check if given method name is deprecated or not.
350	+ */
351	+ public function isDeprecatedMethod($class, $methodName)
352	+ {
353	+ return $this->metadataArray[$class][$methodName]['isDeprecated'];
354	+ }
355	+
356	+ /**
357	+ * Returns the 'moduleName' part of '\\Piwik\\Plugins\\moduleName\\API'
358	+ *
359	+ * @param string $className "API"
360	+ * @return string "Referrers"
361	+ */
362	+ public function getModuleNameFromClassName($className)
363	+ {
364	+ return str_replace(array('\\Piwik\\Plugins\\', '\\API'), '', $className);
365	+ }
366	+
367	+ public function isExistingApiAction($pluginName, $apiAction)
368	+ {
369	+ $namespacedApiClassName = "\\Piwik\\Plugins\\$pluginName\\API";
370	+ $api = $namespacedApiClassName::getInstance();
371	+
372	+ return method_exists($api, $apiAction);
373	+ }
374	+
375	+ public function buildApiActionName($pluginName, $apiAction)
376	+ {
377	+ return sprintf("%s.%s", $pluginName, $apiAction);
378	+ }
379	+
380	+ /**
381	+ * Sets whether to hide '@ignore'd functions from method metadata or not.
382	+ *
383	+ * @param bool $hideIgnoredFunctions
384	+ */
385	+ public function setHideIgnoredFunctions($hideIgnoredFunctions)
386	+ {
387	+ $this->hideIgnoredFunctions = $hideIgnoredFunctions;
388	+
389	+ // make sure metadata gets reloaded
390	+ $this->alreadyRegistered = array();
391	+ $this->metadataArray = array();
392	+ }
393	+
394	+ /**
395	+ * Returns an array containing the values of the parameters to pass to the method to call
396	+ *
397	+ * @param array $requiredParameters array of (parameter name, default value)
398	+ * @param array $parametersRequest
399	+ * @throws Exception
400	+ * @return array values to pass to the function call
401	+ */
402	+ private function getRequestParametersArray($requiredParameters, $parametersRequest)
403	+ {
404	+ $finalParameters = array();
405	+ foreach ($requiredParameters as $name => $defaultValue) {
406	+ try {
407	+ if ($defaultValue instanceof NoDefaultValue) {
408	+ $requestValue = Common::getRequestVar($name, null, null, $parametersRequest);
409	+ } else {
410	+ try {
411	+ if ($name == 'segment' && !empty($parametersRequest['segment'])) {
412	+ // segment parameter is an exception: we do not want to sanitize user input or it would break the segment encoding
413	+ $requestValue = ($parametersRequest['segment']);
414	+ } else {
415	+ $requestValue = Common::getRequestVar($name, $defaultValue, null, $parametersRequest);
416	+ }
417	+ } catch (Exception $e) {
418	+ // Special case: empty parameter in the URL, should return the empty string
419	+ if (isset($parametersRequest[$name])
420	+ && $parametersRequest[$name] === ''
421	+ ) {
422	+ $requestValue = '';
423	+ } else {
424	+ $requestValue = $defaultValue;
425	+ }
426	+ }
427	+ }
428	+ } catch (Exception $e) {
429	+ throw new Exception(Piwik::translate('General_PleaseSpecifyValue', array($name)));
430	+ }
431	+ $finalParameters[$name] = $requestValue;
432	+ }
433	+ return $finalParameters;
434	+ }
435	+
436	+ /**
437	+ * Includes the class API by looking up plugins/xxx/API.php
438	+ *
439	+ * @param string $fileName api class name eg. "API"
440	+ * @throws Exception
441	+ */
442	+ private function includeApiFile($fileName)
443	+ {
444	+ $module = self::getModuleNameFromClassName($fileName);
445	+ $path = Manager::getPluginDirectory($module) . '/API.php';
446	+
447	+ if (is_readable($path)) {
448	+ require_once $path; // prefixed by PIWIK_INCLUDE_PATH
449	+ } else {
450	+ throw new Exception("API module $module not found.");
451	+ }
452	+ }
453	+
454	+ /**
455	+ * @param string $class name of a class
456	+ * @param ReflectionMethod $method instance of ReflectionMethod
457	+ */
458	+ private function loadMethodMetadata($class, $method)
459	+ {
460	+ if (!$this->checkIfMethodIsAvailable($method)) {
461	+ return;
462	+ }
463	+ $name = $method->getName();
464	+ $parameters = $method->getParameters();
465	+ $docComment = $method->getDocComment();
466	+
467	+ $aParameters = array();
468	+ foreach ($parameters as $parameter) {
469	+ $nameVariable = $parameter->getName();
470	+
471	+ $defaultValue = $this->noDefaultValue;
472	+ if ($parameter->isDefaultValueAvailable()) {
473	+ $defaultValue = $parameter->getDefaultValue();
474	+ }
475	+
476	+ $aParameters[$nameVariable] = $defaultValue;
477	+ }
478	+ $this->metadataArray[$class][$name]['parameters'] = $aParameters;
479	+ $this->metadataArray[$class][$name]['numberOfRequiredParameters'] = $method->getNumberOfRequiredParameters();
480	+ $this->metadataArray[$class][$name]['isDeprecated'] = false !== strstr($docComment, '@deprecated');
481	+ }
482	+
483	+ /**
484	+ * Checks that the method exists in the class
485	+ *
486	+ * @param string $className The class name
487	+ * @param string $methodName The method name
488	+ * @throws Exception If the method is not found
489	+ */
490	+ private function checkMethodExists($className, $methodName)
491	+ {
492	+ if (!$this->isMethodAvailable($className, $methodName)) {
493	+ throw new Exception(Piwik::translate('General_ExceptionMethodNotFound', array($methodName, $className)));
494	+ }
495	+ }
496	+
497	+ /**
498	+ * @param $docComment
499	+ * @return bool
500	+ */
501	+ public function shouldHideAPIMethod($docComment)
502	+ {
503	+ $hideLine = strstr($docComment, '@hide');
504	+
505	+ if ($hideLine === false) {
506	+ return false;
507	+ }
508	+
509	+ $hideLine = trim($hideLine);
510	+ $hideLine .= ' ';
511	+
512	+ $token = trim(strtok($hideLine, " "), "\n");
513	+
514	+ $hide = false;
515	+
516	+ if (!empty($token)) {
517	+ /**
518	+ * This event exists for checking whether a Plugin API class or a Plugin API method tagged
519	+ * with a `@hideXYZ` should be hidden in the API listing.
520	+ *
521	+ * @param bool &$hide whether to hide APIs tagged with $token should be displayed.
522	+ */
523	+ Piwik::postEvent(sprintf('API.DocumentationGenerator.%s', $token), array(&$hide));
524	+ }
525	+
526	+ return $hide;
527	+ }
528	+
529	+ /**
530	+ * @param ReflectionMethod $method
531	+ * @return bool
532	+ */
533	+ protected function checkIfMethodIsAvailable(ReflectionMethod $method)
534	+ {
535	+ if (!$method->isPublic() \|\| $method->isConstructor() \|\| $method->getName() === 'getInstance') {
536	+ return false;
537	+ }
538	+
539	+ if ($this->hideIgnoredFunctions && false !== strstr($method->getDocComment(), '@ignore')) {
540	+ return false;
541	+ }
542	+
543	+ if ($this->shouldHideAPIMethod($method->getDocComment())) {
544	+ return false;
545	+ }
546	+
547	+ return true;
548	+ }
549	+
550	+ /**
551	+ * Returns true if the method is found in the API of the given class name.
552	+ *
553	+ * @param string $className The class name
554	+ * @param string $methodName The method name
555	+ * @return bool
556	+ */
557	+ private function isMethodAvailable($className, $methodName)
558	+ {
559	+ return isset($this->metadataArray[$className][$methodName]);
560	+ }
561	+
562	+ /**
563	+ * Checks that the class is a Singleton (presence of the getInstance() method)
564	+ *
565	+ * @param string $className The class name
566	+ * @throws Exception If the class is not a Singleton
567	+ */
568	+ private function checkClassIsSingleton($className)
569	+ {
570	+ if (!method_exists($className, "getInstance")) {
571	+ throw new Exception("$className that provide an API must be Singleton and have a 'public static function getInstance()' method.");
572	+ }
573	+ }
574	+ }
575	+
576	+ /**
577	+ * To differentiate between "no value" and default value of null
578	+ *
579	+ */
580	+ class NoDefaultValue
581	+ {
582	+ }

app/core/API/Request.php ADDED Viewed

	@@ -0,0 +1,653 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik\API;
10	+
11	+ use Exception;
12	+ use Piwik\Access;
13	+ use Piwik\Cache;
14	+ use Piwik\Common;
15	+ use Piwik\Container\StaticContainer;
16	+ use Piwik\Context;
17	+ use Piwik\DataTable;
18	+ use Piwik\Exception\PluginDeactivatedException;
19	+ use Piwik\IP;
20	+ use Piwik\Log;
21	+ use Piwik\Piwik;
22	+ use Piwik\Plugin\Manager as PluginManager;
23	+ use Piwik\Plugins\CoreHome\LoginWhitelist;
24	+ use Piwik\SettingsServer;
25	+ use Piwik\Url;
26	+ use Piwik\UrlHelper;
27	+ use Psr\Log\LoggerInterface;
28	+
29	+ /**
30	+ * Dispatches API requests to the appropriate API method.
31	+ *
32	+ * The Request class is used throughout Piwik to call API methods. The difference
33	+ * between using Request and calling API methods directly is that Request
34	+ * will do more after calling the API including: applying generic filters, applying queued filters,
35	+ * and handling the flat and label query parameters.
36	+ *
37	+ * Additionally, the Request class will forward current query parameters to the request
38	+ * which is more convenient than calling {@link Piwik\Common::getRequestVar()} many times over.
39	+ *
40	+ * In most cases, using a Request object to query the API is the correct approach.
41	+ *
42	+ * ### Post-processing
43	+ *
44	+ * The return value of API methods undergo some extra processing before being returned by Request.
45	+ *
46	+ * ### Output Formats
47	+ *
48	+ * The value returned by Request will be serialized to a certain format before being returned.
49	+ *
50	+ * ### Examples
51	+ *
52	+ * Basic Usage
53	+ *
54	+ * $request = new Request('method=UserLanguage.getLanguage&idSite=1&date=yesterday&period=week'
55	+ * . '&format=xml&filter_limit=5&filter_offset=0')
56	+ * $result = $request->process();
57	+ * echo $result;
58	+ *
59	+ * Getting a unrendered DataTable
60	+ *
61	+ * // use the convenience method 'processRequest'
62	+ * $dataTable = Request::processRequest('UserLanguage.getLanguage', array(
63	+ * 'idSite' => 1,
64	+ * 'date' => 'yesterday',
65	+ * 'period' => 'week',
66	+ * 'filter_limit' => 5,
67	+ * 'filter_offset' => 0
68	+ *
69	+ * 'format' => 'original', // this is the important bit
70	+ * ));
71	+ * echo "This DataTable has " . $dataTable->getRowsCount() . " rows.";
72	+ *
73	+ * @see http://piwik.org/docs/analytics-api
74	+ * @api
75	+ */
76	+ class Request
77	+ {
78	+ /**
79	+ * The count of nested API request invocations. Used to determine if the currently executing request is the root or not.
80	+ *
81	+ * @var int
82	+ */
83	+ private static $nestedApiInvocationCount = 0;
84	+
85	+ private $request = null;
86	+
87	+ /**
88	+ * Converts the supplied request string into an array of query paramater name/value
89	+ * mappings. The current query parameters (everything in `$_GET` and `$_POST`) are
90	+ * forwarded to request array before it is returned.
91	+ *
92	+ * @param string\|array\|null $request The base request string or array, eg,
93	+ * `'module=UserLanguage&action=getLanguage'`.
94	+ * @param array $defaultRequest Default query parameters. If a query parameter is absent in `$request`, it will be loaded
95	+ * from this. Defaults to `$_GET + $_POST`.
96	+ * @return array
97	+ */
98	+ public static function getRequestArrayFromString($request, $defaultRequest = null)
99	+ {
100	+ if ($defaultRequest === null) {
101	+ $defaultRequest = self::getDefaultRequest();
102	+
103	+ $requestRaw = self::getRequestParametersGET();
104	+ if (!empty($requestRaw['segment'])) {
105	+ $defaultRequest['segment'] = $requestRaw['segment'];
106	+ }
107	+
108	+ if (!isset($defaultRequest['format_metrics'])) {
109	+ $defaultRequest['format_metrics'] = 'bc';
110	+ }
111	+ }
112	+
113	+ $requestArray = $defaultRequest;
114	+
115	+ if (!is_null($request)) {
116	+ if (is_array($request)) {
117	+ $requestParsed = $request;
118	+ } else {
119	+ $request = trim($request);
120	+ $request = str_replace(array("\n", "\t"), '', $request);
121	+
122	+ $requestParsed = UrlHelper::getArrayFromQueryString($request);
123	+ }
124	+
125	+ $requestArray = $requestParsed + $defaultRequest;
126	+ }
127	+
128	+ foreach ($requestArray as &$element) {
129	+ if (!is_array($element)) {
130	+ $element = trim($element);
131	+ }
132	+ }
133	+ return $requestArray;
134	+ }
135	+
136	+ /**
137	+ * Constructor.
138	+ *
139	+ * @param string\|array $request Query string that defines the API call (must at least contain a method parameter),
140	+ * eg, `'method=UserLanguage.getLanguage&idSite=1&date=yesterday&period=week&format=xml'`
141	+ * If a request is not provided, then we use the values in the `$_GET` and `$_POST`
142	+ * superglobals.
143	+ * @param array $defaultRequest Default query parameters. If a query parameter is absent in `$request`, it will be loaded
144	+ * from this. Defaults to `$_GET + $_POST`.
145	+ */
146	+ public function __construct($request = null, $defaultRequest = null)
147	+ {
148	+ $this->request = self::getRequestArrayFromString($request, $defaultRequest);
149	+ $this->sanitizeRequest();
150	+ $this->renameModuleAndActionInRequest();
151	+ }
152	+
153	+ /**
154	+ * For backward compatibility: Piwik API still works if module=Referers,
155	+ * we rewrite to correct renamed plugin: Referrers
156	+ *
157	+ * @param $module
158	+ * @param $action
159	+ * @return array( $module, $action )
160	+ * @ignore
161	+ */
162	+ public static function getRenamedModuleAndAction($module, $action)
163	+ {
164	+ /**
165	+ * This event is posted in the Request dispatcher and can be used
166	+ * to overwrite the Module and Action to dispatch.
167	+ * This is useful when some Controller methods or API methods have been renamed or moved to another plugin.
168	+ *
169	+ * @param $module string
170	+ * @param $action string
171	+ */
172	+ Piwik::postEvent('Request.getRenamedModuleAndAction', array(&$module, &$action));
173	+
174	+ return array($module, $action);
175	+ }
176	+
177	+ /**
178	+ * Make sure that the request contains no logical errors
179	+ */
180	+ private function sanitizeRequest()
181	+ {
182	+ // The label filter does not work with expanded=1 because the data table IDs have a different meaning
183	+ // depending on whether the table has been loaded yet. expanded=1 causes all tables to be loaded, which
184	+ // is why the label filter can't descend when a recursive label has been requested.
185	+ // To fix this, we remove the expanded parameter if a label parameter is set.
186	+ if (isset($this->request['label']) && !empty($this->request['label'])
187	+ && isset($this->request['expanded']) && $this->request['expanded']
188	+ ) {
189	+ unset($this->request['expanded']);
190	+ }
191	+ }
192	+
193	+ /**
194	+ * Dispatches the API request to the appropriate API method and returns the result
195	+ * after post-processing.
196	+ *
197	+ * Post-processing includes:
198	+ *
199	+ * - flattening if flat is 0
200	+ * - running generic filters unless disable_generic_filters is set to 1
201	+ * - URL decoding label column values
202	+ * - running queued filters unless disable_queued_filters is set to 1
203	+ * - removing columns based on the values of the hideColumns and showColumns query parameters
204	+ * - filtering rows if the label query parameter is set
205	+ * - converting the result to the appropriate format (ie, XML, JSON, etc.)
206	+ *
207	+ * If `'original'` is supplied for the output format, the result is returned as a PHP
208	+ * object.
209	+ *
210	+ * @throws PluginDeactivatedException if the module plugin is not activated.
211	+ * @throws Exception if the requested API method cannot be called, if required parameters for the
212	+ * API method are missing or if the API method throws an exception and the format
213	+ * query parameter is original.
214	+ * @return DataTable\|Map\|string The data resulting from the API call.
215	+ */
216	+ public function process()
217	+ {
218	+ try {
219	+ ++self::$nestedApiInvocationCount;
220	+
221	+ // read the format requested for the output data
222	+ $outputFormat = strtolower(Common::getRequestVar('format', 'xml', 'string', $this->request));
223	+
224	+ $disablePostProcessing = $this->shouldDisablePostProcessing();
225	+
226	+ // create the response
227	+ $response = new ResponseBuilder($outputFormat, $this->request);
228	+ if ($disablePostProcessing) {
229	+ $response->disableDataTablePostProcessor();
230	+ }
231	+
232	+ $corsHandler = new CORSHandler();
233	+ $corsHandler->handle();
234	+
235	+ $tokenAuth = Common::getRequestVar('token_auth', '', 'string', $this->request);
236	+ $shouldReloadAuth = false;
237	+
238	+ // IP check is needed here as we cannot listen to API.Request.authenticate as it would then not return proper API format response.
239	+ // We can also not do it by listening to API.Request.dispatch as by then the user is already authenticated and we want to make sure
240	+ // to not expose any information in case the IP is not whitelisted.
241	+ $whitelist = new LoginWhitelist();
242	+ if ($whitelist->shouldCheckWhitelist() && $whitelist->shouldWhitelistApplyToAPI()) {
243	+ $ip = IP::getIpFromHeader();
244	+ $whitelist->checkIsWhitelisted($ip);
245	+ }
246	+
247	+ // read parameters
248	+ $moduleMethod = Common::getRequestVar('method', null, 'string', $this->request);
249	+
250	+ list($module, $method) = $this->extractModuleAndMethod($moduleMethod);
251	+ list($module, $method) = self::getRenamedModuleAndAction($module, $method);
252	+
253	+ PluginManager::getInstance()->checkIsPluginActivated($module);
254	+
255	+ $apiClassName = self::getClassNameAPI($module);
256	+
257	+ if ($shouldReloadAuth = self::shouldReloadAuthUsingTokenAuth($this->request)) {
258	+ $access = Access::getInstance();
259	+ $tokenAuthToRestore = $access->getTokenAuth();
260	+ $hadSuperUserAccess = $access->hasSuperUserAccess();
261	+ self::forceReloadAuthUsingTokenAuth($tokenAuth);
262	+ }
263	+
264	+ // call the method
265	+ $returnedValue = Proxy::getInstance()->call($apiClassName, $method, $this->request);
266	+
267	+ // get the response with the request query parameters loaded, since DataTablePost processor will use the Report
268	+ // class instance, which may inspect the query parameters. (eg, it may look for the idCustomReport parameters
269	+ // which may only exist in $this->request, if the request was called programatically)
270	+ $toReturn = Context::executeWithQueryParameters($this->request, function () use ($response, $returnedValue, $module, $method) {
271	+ return $response->getResponse($returnedValue, $module, $method);
272	+ });
273	+ } catch (Exception $e) {
274	+ StaticContainer::get(LoggerInterface::class)->error('Uncaught exception in API: {exception}', [
275	+ 'exception' => $e,
276	+ 'ignoreInScreenWriter' => true,
277	+ ]);
278	+
279	+ $toReturn = $response->getResponseException($e);
280	+ } finally {
281	+ --self::$nestedApiInvocationCount;
282	+ }
283	+
284	+ if ($shouldReloadAuth) {
285	+ $this->restoreAuthUsingTokenAuth($tokenAuthToRestore, $hadSuperUserAccess);
286	+ }
287	+
288	+ return $toReturn;
289	+ }
290	+
291	+ private function restoreAuthUsingTokenAuth($tokenToRestore, $hadSuperUserAccess)
292	+ {
293	+ // if we would not make sure to unset super user access, the tokenAuth would be not authenticated and any
294	+ // token would just keep super user access (eg if the token that was reloaded before had super user access)
295	+ Access::getInstance()->setSuperUserAccess(false);
296	+
297	+ // we need to restore by reloading the tokenAuth as some permissions could have been removed in the API
298	+ // request etc. Otherwise we could just store a clone of Access::getInstance() and restore here
299	+ self::forceReloadAuthUsingTokenAuth($tokenToRestore);
300	+
301	+ if ($hadSuperUserAccess && !Access::getInstance()->hasSuperUserAccess()) {
302	+ // we are in context of `doAsSuperUser()` and need to restore this behaviour
303	+ Access::getInstance()->setSuperUserAccess(true);
304	+ }
305	+ }
306	+
307	+ /**
308	+ * Returns the name of a plugin's API class by plugin name.
309	+ *
310	+ * @param string $plugin The plugin name, eg, `'Referrers'`.
311	+ * @return string The fully qualified API class name, eg, `'\Piwik\Plugins\Referrers\API'`.
312	+ */
313	+ public static function getClassNameAPI($plugin)
314	+ {
315	+ return sprintf('\Piwik\Plugins\%s\API', $plugin);
316	+ }
317	+
318	+ /**
319	+ * @ignore
320	+ * @internal
321	+ * @param string $currentApiMethod
322	+ */
323	+ public static function setIsRootRequestApiRequest($currentApiMethod)
324	+ {
325	+ Cache::getTransientCache()->save('API.setIsRootRequestApiRequest', $currentApiMethod);
326	+ }
327	+
328	+ /**
329	+ * @ignore
330	+ * @internal
331	+ * @return string current Api Method if it is an api request
332	+ */
333	+ public static function getRootApiRequestMethod()
334	+ {
335	+ return Cache::getTransientCache()->fetch('API.setIsRootRequestApiRequest');
336	+ }
337	+
338	+ /**
339	+ * Detect if the root request (the actual request) is an API request or not. To detect whether an API is currently
340	+ * request within any request, have a look at {@link isApiRequest()}.
341	+ *
342	+ * @return bool
343	+ * @throws Exception
344	+ */
345	+ public static function isRootRequestApiRequest()
346	+ {
347	+ $apiMethod = Cache::getTransientCache()->fetch('API.setIsRootRequestApiRequest');
348	+ return !empty($apiMethod);
349	+ }
350	+
351	+ /**
352	+ * Checks if the currently executing API request is the root API request or not.
353	+ *
354	+ * Note: the "root" API request is the first request made. Within that request, further API methods
355	+ * can be called programmatically. These requests are considered "child" API requests.
356	+ *
357	+ * @return bool
358	+ * @throws Exception
359	+ */
360	+ public static function isCurrentApiRequestTheRootApiRequest()
361	+ {
362	+ return self::$nestedApiInvocationCount == 1;
363	+ }
364	+
365	+ /**
366	+ * Detect if request is an API request. Meaning the module is 'API' and an API method having a valid format was
367	+ * specified. Note that this method will return true even if the actual request is for example a regular UI
368	+ * reporting page request but within this request we are currently processing an API request (eg a
369	+ * controller calls Request::processRequest('API.getMatomoVersion')). To find out if the root request is an API
370	+ * request or not, call {@link isRootRequestApiRequest()}
371	+ *
372	+ * @param array $request eg array('module' => 'API', 'method' => 'Test.getMethod')
373	+ * @return bool
374	+ * @throws Exception
375	+ */
376	+ public static function isApiRequest($request)
377	+ {
378	+ $method = self::getMethodIfApiRequest($request);
379	+ return !empty($method);
380	+ }
381	+
382	+ /**
383	+ * Returns the current API method being executed, if the current request is an API request.
384	+ *
385	+ * @param array $request eg array('module' => 'API', 'method' => 'Test.getMethod')
386	+ * @return string\|null
387	+ * @throws Exception
388	+ */
389	+ public static function getMethodIfApiRequest($request)
390	+ {
391	+ $module = Common::getRequestVar('module', '', 'string', $request);
392	+ $method = Common::getRequestVar('method', '', 'string', $request);
393	+
394	+ $isApi = $module === 'API' && !empty($method) && (count(explode('.', $method)) === 2);
395	+ return $isApi ? $method : null;
396	+ }
397	+
398	+ /**
399	+ * If the token_auth is found in the $request parameter,
400	+ * the current session will be authenticated using this token_auth.
401	+ * It will overwrite the previous Auth object.
402	+ *
403	+ * @param array $request If null, uses the default request ($_GET)
404	+ * @return void
405	+ * @ignore
406	+ */
407	+ public static function reloadAuthUsingTokenAuth($request = null)
408	+ {
409	+ // if a token_auth is specified in the API request, we load the right permissions
410	+ $token_auth = Common::getRequestVar('token_auth', '', 'string', $request);
411	+
412	+ if (self::shouldReloadAuthUsingTokenAuth($request)) {
413	+ self::forceReloadAuthUsingTokenAuth($token_auth);
414	+ }
415	+ }
416	+
417	+ /**
418	+ * The current session will be authenticated using this token_auth.
419	+ * It will overwrite the previous Auth object.
420	+ *
421	+ * @param string $tokenAuth
422	+ * @return void
423	+ */
424	+ private static function forceReloadAuthUsingTokenAuth($tokenAuth)
425	+ {
426	+ /**
427	+ * Triggered when authenticating an API request, but only if the token_auth
428	+ * query parameter is found in the request.
429	+ *
430	+ * Plugins that provide authentication capabilities should subscribe to this event
431	+ * and make sure the global authentication object (the object returned by `StaticContainer::get('Piwik\Auth')`)
432	+ * is setup to use `$token_auth` when its `authenticate()` method is executed.
433	+ *
434	+ * @param string $token_auth The value of the token_auth query parameter.
435	+ */
436	+ Piwik::postEvent('API.Request.authenticate', array($tokenAuth));
437	+ if (!Access::getInstance()->reloadAccess() && $tokenAuth && $tokenAuth !== 'anonymous') {
438	+ /**
439	+ * @ignore
440	+ * @internal
441	+ */
442	+ Piwik::postEvent('API.Request.authenticate.failed');
443	+ }
444	+ SettingsServer::raiseMemoryLimitIfNecessary();
445	+ }
446	+
447	+ private static function shouldReloadAuthUsingTokenAuth($request)
448	+ {
449	+ if (is_null($request)) {
450	+ $request = self::getDefaultRequest();
451	+ }
452	+
453	+ if (!isset($request['token_auth'])) {
454	+ // no token is given so we just keep the current loaded user
455	+ return false;
456	+ }
457	+
458	+ // a token is specified, we need to reload auth in case it is different than the current one, even if it is empty
459	+ $tokenAuth = Common::getRequestVar('token_auth', '', 'string', $request);
460	+
461	+ // not using !== is on purpose as getTokenAuth() might return null whereas $tokenAuth is '' . In this case
462	+ // we do not need to reload.
463	+
464	+ return $tokenAuth != Access::getInstance()->getTokenAuth();
465	+ }
466	+
467	+ /**
468	+ * Returns array($class, $method) from the given string $class.$method
469	+ *
470	+ * @param string $parameter
471	+ * @throws Exception
472	+ * @return array
473	+ */
474	+ private function extractModuleAndMethod($parameter)
475	+ {
476	+ $a = explode('.', $parameter);
477	+ if (count($a) != 2) {
478	+ throw new Exception("The method name is invalid. Expected 'module.methodName'");
479	+ }
480	+ return $a;
481	+ }
482	+
483	+ /**
484	+ * Helper method that processes an API request in one line using the variables in `$_GET`
485	+ * and `$_POST`.
486	+ *
487	+ * @param string $method The API method to call, ie, `'Actions.getPageTitles'`.
488	+ * @param array $paramOverride The parameter name-value pairs to use instead of what's
489	+ * in `$_GET` & `$_POST`.
490	+ * @param array $defaultRequest Default query parameters. If a query parameter is absent in `$request`, it will be loaded
491	+ * from this. Defaults to `$_GET + $_POST`.
492	+ *
493	+ * To avoid using any parameters from $_GET or $_POST, set this to an empty `array()`.
494	+ * @return mixed The result of the API request. See {@link process()}.
495	+ */
496	+ public static function processRequest($method, $paramOverride = array(), $defaultRequest = null)
497	+ {
498	+ $params = array();
499	+ $params['format'] = 'original';
500	+ $params['serialize'] = '0';
501	+ $params['module'] = 'API';
502	+ $params['method'] = $method;
503	+ $params['compare'] = '0';
504	+ $params = $paramOverride + $params;
505	+
506	+ // process request
507	+ $request = new Request($params, $defaultRequest);
508	+ return $request->process();
509	+ }
510	+
511	+ /**
512	+ * Returns the original request parameters in the current query string as an array mapping
513	+ * query parameter names with values. The result of this function will not be affected
514	+ * by any modifications to `$_GET` and will not include parameters in `$_POST`.
515	+ *
516	+ * @return array
517	+ */
518	+ public static function getRequestParametersGET()
519	+ {
520	+ if (empty($_SERVER['QUERY_STRING'])) {
521	+ return array();
522	+ }
523	+ $GET = UrlHelper::getArrayFromQueryString($_SERVER['QUERY_STRING']);
524	+ return $GET;
525	+ }
526	+
527	+ /**
528	+ * Returns the URL for the current requested report w/o any filter parameters.
529	+ *
530	+ * @param string $module The API module.
531	+ * @param string $action The API action.
532	+ * @param array $queryParams Query parameter overrides.
533	+ * @return string
534	+ */
535	+ public static function getBaseReportUrl($module, $action, $queryParams = array())
536	+ {
537	+ $params = array_merge($queryParams, array('module' => $module, 'action' => $action));
538	+ return Request::getCurrentUrlWithoutGenericFilters($params);
539	+ }
540	+
541	+ /**
542	+ * Returns the current URL without generic filter query parameters.
543	+ *
544	+ * @param array $params Query parameter values to override in the new URL.
545	+ * @return string
546	+ */
547	+ public static function getCurrentUrlWithoutGenericFilters($params)
548	+ {
549	+ // unset all filter query params so the related report will show up in its default state,
550	+ // unless the filter param was in $queryParams
551	+ $genericFiltersInfo = DataTableGenericFilter::getGenericFiltersInformation();
552	+ foreach ($genericFiltersInfo as $filter) {
553	+ foreach ($filter[1] as $queryParamName => $queryParamInfo) {
554	+ if (!isset($params[$queryParamName])) {
555	+ $params[$queryParamName] = null;
556	+ }
557	+ }
558	+ }
559	+
560	+ $params['compareDates'] = null;
561	+ $params['comparePeriods'] = null;
562	+ $params['compareSegments'] = null;
563	+
564	+ return Url::getCurrentQueryStringWithParametersModified($params);
565	+ }
566	+
567	+ /**
568	+ * Returns whether the DataTable result will have to be expanded for the
569	+ * current request before rendering.
570	+ *
571	+ * @return bool
572	+ * @ignore
573	+ */
574	+ public static function shouldLoadExpanded()
575	+ {
576	+ // if filter_column_recursive & filter_pattern_recursive are supplied, and flat isn't supplied
577	+ // we have to load all the child subtables.
578	+ return Common::getRequestVar('filter_column_recursive', false) !== false
579	+ && Common::getRequestVar('filter_pattern_recursive', false) !== false
580	+ && !self::shouldLoadFlatten();
581	+ }
582	+
583	+ /**
584	+ * @return bool
585	+ */
586	+ public static function shouldLoadFlatten()
587	+ {
588	+ return Common::getRequestVar('flat', false) == 1;
589	+ }
590	+
591	+ /**
592	+ * Returns the segment query parameter from the original request, without modifications.
593	+ *
594	+ * @return array\|bool
595	+ */
596	+ public static function getRawSegmentFromRequest()
597	+ {
598	+ // we need the URL encoded segment parameter, we fetch it from _SERVER['QUERY_STRING'] instead of default URL decoded _GET
599	+ $segmentRaw = false;
600	+ $segment = Common::getRequestVar('segment', '', 'string');
601	+ if (!empty($segment)) {
602	+ $request = Request::getRequestParametersGET();
603	+ if (!empty($request['segment'])) {
604	+ $segmentRaw = $request['segment'];
605	+ }
606	+ }
607	+ return $segmentRaw;
608	+ }
609	+
610	+ private function renameModuleAndActionInRequest()
611	+ {
612	+ if (empty($this->request['apiModule'])) {
613	+ return;
614	+ }
615	+ if (empty($this->request['apiAction'])) {
616	+ $this->request['apiAction'] = null;
617	+ }
618	+ list($this->request['apiModule'], $this->request['apiAction']) = $this->getRenamedModuleAndAction($this->request['apiModule'], $this->request['apiAction']);
619	+ }
620	+
621	+ /**
622	+ * @return array
623	+ */
624	+ private static function getDefaultRequest()
625	+ {
626	+ return $_GET + $_POST;
627	+ }
628	+
629	+ private function shouldDisablePostProcessing()
630	+ {
631	+ $shouldDisable = false;
632	+
633	+ /**
634	+ * After an API method returns a value, the value is post processed (eg, rows are sorted
635	+ * based on the `filter_sort_column` query parameter, rows are truncated based on the
636	+ * `filter_limit`/`filter_offset` parameters, amongst other things).
637	+ *
638	+ * If you're creating a plugin that needs to disable post processing entirely for
639	+ * certain requests, use this event.
640	+ *
641	+ * @param bool &$shouldDisable Set this to true to disable datatable post processing for a request.
642	+ * @param array $request The request parameters.
643	+ */
644	+ Piwik::postEvent('Request.shouldDisablePostProcessing', [&$shouldDisable, $this->request]);
645	+
646	+ if (!$shouldDisable) {
647	+ $shouldDisable = self::isCurrentApiRequestTheRootApiRequest() &&
648	+ Common::getRequestVar('disable_root_datatable_post_processor', 0, 'int', $this->request) == 1;
649	+ }
650	+
651	+ return $shouldDisable;
652	+ }
653	+ }

app/core/API/ResponseBuilder.php ADDED Viewed

	@@ -0,0 +1,245 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik\API;
10	+
11	+ use Exception;
12	+ use Piwik\Common;
13	+ use Piwik\DataTable;
14	+ use Piwik\DataTable\Renderer;
15	+ use Piwik\DataTable\DataTableInterface;
16	+ use Piwik\DataTable\Filter\ColumnDelete;
17	+ use Piwik\DataTable\Filter\Pattern;
18	+ use Piwik\Http\HttpCodeException;
19	+ use Piwik\Plugins\Monolog\Processor\ExceptionToTextProcessor;
20	+
21	+ /**
22	+ */
23	+ class ResponseBuilder
24	+ {
25	+ private $outputFormat = null;
26	+ private $apiRenderer = null;
27	+ private $request = null;
28	+ private $sendHeader = true;
29	+ private $postProcessDataTable = true;
30	+
31	+ private $apiModule = false;
32	+ private $apiMethod = false;
33	+ private $shouldPrintBacktrace = false;
34	+
35	+ /**
36	+ * @param string $outputFormat
37	+ * @param array $request
38	+ */
39	+ public function __construct($outputFormat, $request = array(), $shouldPrintBacktrace = null)
40	+ {
41	+ $this->outputFormat = $outputFormat;
42	+ $this->request = $request;
43	+ $this->apiRenderer = ApiRenderer::factory($outputFormat, $request);
44	+ $this->shouldPrintBacktrace = $shouldPrintBacktrace === null ? \Piwik_ShouldPrintBackTraceWithMessage() : $shouldPrintBacktrace;
45	+ }
46	+
47	+ public function disableSendHeader()
48	+ {
49	+ $this->sendHeader = false;
50	+ }
51	+
52	+ public function disableDataTablePostProcessor()
53	+ {
54	+ $this->postProcessDataTable = false;
55	+ }
56	+
57	+ /**
58	+ * This method processes the data resulting from the API call.
59	+ *
60	+ * - If the data resulted from the API call is a DataTable then
61	+ * - we apply the standard filters if the parameters have been found
62	+ * in the URL. For example to offset,limit the Table you can add the following parameters to any API
63	+ * call that returns a DataTable: filter_limit=10&filter_offset=20
64	+ * - we apply the filters that have been previously queued on the DataTable
65	+ * @see DataTable::queueFilter()
66	+ * - we apply the renderer that generate the DataTable in a given format (XML, PHP, HTML, JSON, etc.)
67	+ * the format can be changed using the 'format' parameter in the request.
68	+ * Example: format=xml
69	+ *
70	+ * - If there is nothing returned (void) we display a standard success message
71	+ *
72	+ * - If there is a PHP array returned, we try to convert it to a dataTable
73	+ * It is then possible to convert this datatable to any requested format (xml/etc)
74	+ *
75	+ * - If a bool is returned we convert to a string (true is displayed as 'true' false as 'false')
76	+ *
77	+ * - If an integer / float is returned, we simply return it
78	+ *
79	+ * @param mixed $value The initial returned value, before post process. If set to null, success response is returned.
80	+ * @param bool\|string $apiModule The API module that was called
81	+ * @param bool\|string $apiMethod The API method that was called
82	+ * @return mixed Usually a string, but can still be a PHP data structure if the format requested is 'original'
83	+ */
84	+ public function getResponse($value = null, $apiModule = false, $apiMethod = false)
85	+ {
86	+ $this->apiModule = $apiModule;
87	+ $this->apiMethod = $apiMethod;
88	+
89	+ $this->sendHeaderIfEnabled();
90	+
91	+ // when null or void is returned from the api call, we handle it as a successful operation
92	+ if (!isset($value)) {
93	+ if (ob_get_contents()) {
94	+ return null;
95	+ }
96	+
97	+ return $this->apiRenderer->renderSuccess('ok');
98	+ }
99	+
100	+ // If the returned value is an object DataTable we
101	+ // apply the set of generic filters if asked in the URL
102	+ // and we render the DataTable according to the format specified in the URL
103	+ if ($value instanceof DataTableInterface) {
104	+ return $this->handleDataTable($value);
105	+ }
106	+
107	+ // Case an array is returned from the API call, we convert it to the requested format
108	+ // - if calling from inside the application (format = original)
109	+ // => the data stays unchanged (ie. a standard php array or whatever data structure)
110	+ // - if any other format is requested, we have to convert this data structure (which we assume
111	+ // to be an array) to a DataTable in order to apply the requested DataTable_Renderer (for example XML)
112	+ if (is_array($value)) {
113	+ return $this->handleArray($value);
114	+ }
115	+
116	+ if (is_object($value)) {
117	+ return $this->apiRenderer->renderObject($value);
118	+ }
119	+
120	+ if (is_resource($value)) {
121	+ return $this->apiRenderer->renderResource($value);
122	+ }
123	+
124	+ return $this->apiRenderer->renderScalar($value);
125	+ }
126	+
127	+ /**
128	+ * Returns an error $message in the requested $format
129	+ *
130	+ * @param Exception\|\Throwable $e
131	+ * @throws Exception
132	+ * @return string
133	+ */
134	+ public function getResponseException($e)
135	+ {
136	+ $e = $this->decorateExceptionWithDebugTrace($e);
137	+ $message = $this->formatExceptionMessage($e);
138	+
139	+ if ($this->sendHeader
140	+ && $e instanceof HttpCodeException
141	+ && $e->getCode() > 0
142	+ ) {
143	+ http_response_code($e->getCode());
144	+ }
145	+
146	+ $this->sendHeaderIfEnabled();
147	+
148	+ return $this->apiRenderer->renderException($message, $e);
149	+ }
150	+
151	+ /**
152	+ * @param Exception\|\Throwable $e
153	+ * @return Exception
154	+ */
155	+ private function decorateExceptionWithDebugTrace($e)
156	+ {
157	+ // If we are in tests, show full backtrace
158	+ if (defined('PIWIK_PATH_TEST_TO_ROOT')) {
159	+ if ($this->shouldPrintBacktrace) {
160	+ $message = $e->getMessage() . " in \n " . $e->getFile() . ":" . $e->getLine() . " \n " . $e->getTraceAsString();
161	+ } else {
162	+ $message = $e->getMessage() . "\n \n --> To temporarily debug this error further, set const PIWIK_PRINT_ERROR_BACKTRACE=true; in index.php";
163	+ }
164	+
165	+ return new Exception($message);
166	+ }
167	+
168	+ return $e;
169	+ }
170	+
171	+ /**
172	+ * @param Exception\|\Throwable $exception
173	+ * @return string
174	+ */
175	+ private function formatExceptionMessage($exception)
176	+ {
177	+ $message = ExceptionToTextProcessor::getWholeBacktrace($exception, $this->shouldPrintBacktrace);
178	+
179	+ if ($exception instanceof \Piwik\Exception\Exception && $exception->isHtmlMessage() && Request::isRootRequestApiRequest()) {
180	+ $message = strip_tags(str_replace('<br />', PHP_EOL, $message));
181	+ }
182	+
183	+ return Renderer::formatValueXml($message);
184	+ }
185	+
186	+ private function handleDataTable(DataTableInterface $datatable)
187	+ {
188	+ if ($this->postProcessDataTable) {
189	+ $postProcessor = new DataTablePostProcessor($this->apiModule, $this->apiMethod, $this->request);
190	+ $datatable = $postProcessor->process($datatable);
191	+ }
192	+
193	+ return $this->apiRenderer->renderDataTable($datatable);
194	+ }
195	+
196	+ private function handleArray($array)
197	+ {
198	+ $firstArray = null;
199	+ $firstKey = null;
200	+ if (!empty($array)) {
201	+ $firstArray = reset($array);
202	+ $firstKey = key($array);
203	+ }
204	+
205	+ $isAssoc = !empty($firstArray) && is_numeric($firstKey) && is_array($firstArray) && count(array_filter(array_keys($firstArray), 'is_string'));
206	+
207	+ if (is_numeric($firstKey)) {
208	+ $columns = Common::getRequestVar('filter_column', false, 'array', $this->request);
209	+ $pattern = Common::getRequestVar('filter_pattern', '', 'string', $this->request);
210	+
211	+ if ($columns != array(false) && $pattern !== '') {
212	+ $pattern = new Pattern(new DataTable(), $columns, $pattern);
213	+ $array = $pattern->filterArray($array);
214	+ }
215	+
216	+ $limit = Common::getRequestVar('filter_limit', -1, 'integer', $this->request);
217	+ $offset = Common::getRequestVar('filter_offset', '0', 'integer', $this->request);
218	+
219	+ if ($limit >= 0 \|\| $offset > 0) {
220	+ if ($limit < 0) {
221	+ $limit = null; // make sure to return all results from offset
222	+ }
223	+ $array = array_slice($array, $offset, $limit, $preserveKeys = false);
224	+ }
225	+ }
226	+
227	+ if ($isAssoc) {
228	+ $hideColumns = Common::getRequestVar('hideColumns', '', 'string', $this->request);
229	+ $showColumns = Common::getRequestVar('showColumns', '', 'string', $this->request);
230	+ if ($hideColumns !== '' \|\| $showColumns !== '') {
231	+ $columnDelete = new ColumnDelete(new DataTable(), $hideColumns, $showColumns);
232	+ $array = $columnDelete->filter($array);
233	+ }
234	+ }
235	+
236	+ return $this->apiRenderer->renderArray($array);
237	+ }
238	+
239	+ private function sendHeaderIfEnabled()
240	+ {
241	+ if ($this->sendHeader) {
242	+ $this->apiRenderer->sendHeader();
243	+ }
244	+ }
245	+ }

app/core/Access.php ADDED Viewed

	@@ -0,0 +1,739 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik;
10	+
11	+ use Exception;
12	+ use Piwik\Access\CapabilitiesProvider;
13	+ use Piwik\API\Request;
14	+ use Piwik\Access\RolesProvider;
15	+ use Piwik\Container\StaticContainer;
16	+ use Piwik\Exception\InvalidRequestParameterException;
17	+ use Piwik\Plugins\SitesManager\API as SitesManagerApi;
18	+
19	+ /**
20	+ * Singleton that manages user access to Piwik resources.
21	+ *
22	+ * To check whether a user has access to a resource, use one of the {@link Piwik Piwik::checkUser...}
23	+ * methods.
24	+ *
25	+ * In Piwik there are four different access levels:
26	+ *
27	+ * - no access: Users with this access level cannot view the resource.
28	+ * - view access: Users with this access level can view the resource, but cannot modify it.
29	+ * - admin access: Users with this access level can view and modify the resource.
30	+ * - Super User access: Only the Super User has this access level. It means the user can do
31	+ * whatever they want.
32	+ *
33	+ * Super user access is required to set some configuration options.
34	+ * All other options are specific to the user or to a website.
35	+ *
36	+ * Access is granted per website. Uses with access for a website can view all
37	+ * data associated with that website.
38	+ *
39	+ */
40	+ class Access
41	+ {
42	+ /**
43	+ * Array of idsites available to the current user, indexed by permission level
44	+ * @see getSitesIdWith*()
45	+ *
46	+ * @var array
47	+ */
48	+ protected $idsitesByAccess = null;
49	+
50	+ /**
51	+ * Login of the current user
52	+ *
53	+ * @var string
54	+ */
55	+ protected $login = null;
56	+
57	+ /**
58	+ * token_auth of the current user
59	+ *
60	+ * @var string
61	+ */
62	+ protected $token_auth = null;
63	+
64	+ /**
65	+ * Defines if the current user is the Super User
66	+ * @see hasSuperUserAccess()
67	+ *
68	+ * @var bool
69	+ */
70	+ protected $hasSuperUserAccess = false;
71	+
72	+ /**
73	+ * Authentification object (see Auth)
74	+ *
75	+ * @var Auth
76	+ */
77	+ private $auth = null;
78	+
79	+ /**
80	+ * Gets the singleton instance. Creates it if necessary.
81	+ *
82	+ * @return self
83	+ */
84	+ public static function getInstance()
85	+ {
86	+ return StaticContainer::get('Piwik\Access');
87	+ }
88	+
89	+ /**
90	+ * @var CapabilitiesProvider
91	+ */
92	+ protected $capabilityProvider;
93	+
94	+ /**
95	+ * @var RolesProvider
96	+ */
97	+ private $roleProvider;
98	+
99	+ /**
100	+ * Constructor
101	+ */
102	+ public function __construct(RolesProvider $roleProvider = null, CapabilitiesProvider $capabilityProvider = null)
103	+ {
104	+ if (!isset($roleProvider)) {
105	+ $roleProvider = StaticContainer::get('Piwik\Access\RolesProvider');
106	+ }
107	+ if (!isset($capabilityProvider)) {
108	+ $capabilityProvider = StaticContainer::get('Piwik\Access\CapabilitiesProvider');
109	+ }
110	+ $this->roleProvider = $roleProvider;
111	+ $this->capabilityProvider = $capabilityProvider;
112	+
113	+ $this->resetSites();
114	+ }
115	+
116	+ private function resetSites()
117	+ {
118	+ $this->idsitesByAccess = array(
119	+ 'view' => array(),
120	+ 'write' => array(),
121	+ 'admin' => array(),
122	+ 'superuser' => array()
123	+ );
124	+ }
125	+
126	+ /**
127	+ * Loads the access levels for the current user.
128	+ *
129	+ * Calls the authentication method to try to log the user in the system.
130	+ * If the user credentials are not correct we don't load anything.
131	+ * If the login/password is correct the user is either the SuperUser or a normal user.
132	+ * We load the access levels for this user for all the websites.
133	+ *
134	+ * @param null\|Auth $auth Auth adapter
135	+ * @return bool true on success, false if reloading access failed (when auth object wasn't specified and user is not enforced to be Super User)
136	+ */
137	+ public function reloadAccess(Auth $auth = null)
138	+ {
139	+ $this->resetSites();
140	+
141	+ if (isset($auth)) {
142	+ $this->auth = $auth;
143	+ }
144	+
145	+ if ($this->hasSuperUserAccess()) {
146	+ $this->makeSureLoginNameIsSet();
147	+ return true;
148	+ }
149	+
150	+ $this->token_auth = null;
151	+ $this->login = null;
152	+
153	+ // if the Auth wasn't set, we may be in the special case of setSuperUser(), otherwise we fail TODO: docs + review
154	+ if (!isset($this->auth)) {
155	+ return false;
156	+ }
157	+
158	+ // access = array ( idsite => accessIdSite, idsite2 => accessIdSite2)
159	+ $result = $this->auth->authenticate();
160	+
161	+ if (!$result->wasAuthenticationSuccessful()) {
162	+ return false;
163	+ }
164	+
165	+ $this->login = $result->getIdentity();
166	+ $this->token_auth = $result->getTokenAuth();
167	+
168	+ // case the superUser is logged in
169	+ if ($result->hasSuperUserAccess()) {
170	+ $this->setSuperUserAccess(true);
171	+ }
172	+
173	+ return true;
174	+ }
175	+
176	+ public function getRawSitesWithSomeViewAccess($login)
177	+ {
178	+ $sql = self::getSqlAccessSite("access, t2.idsite");
179	+
180	+ return Db::fetchAll($sql, $login);
181	+ }
182	+
183	+ /**
184	+ * Returns the SQL query joining sites and access table for a given login
185	+ *
186	+ * @param string $select Columns or expression to SELECT FROM table, eg. "MIN(ts_created)"
187	+ * @return string SQL query
188	+ */
189	+ public static function getSqlAccessSite($select)
190	+ {
191	+ $access = Common::prefixTable('access');
192	+ $siteTable = Common::prefixTable('site');
193	+
194	+ return "SELECT " . $select . " FROM " . $access . " as t1
195	+ JOIN " . $siteTable . " as t2 USING (idsite) WHERE login = ?";
196	+ }
197	+
198	+ /**
199	+ * Make sure a login name is set
200	+ *
201	+ * @return true
202	+ */
203	+ protected function makeSureLoginNameIsSet()
204	+ {
205	+ if (empty($this->login)) {
206	+ // flag to force non empty login so Super User is not mistaken for anonymous
207	+ $this->login = 'super user was set';
208	+ }
209	+ }
210	+
211	+ protected function loadSitesIfNeeded()
212	+ {
213	+ if ($this->hasSuperUserAccess) {
214	+ if (empty($this->idsitesByAccess['superuser'])) {
215	+ try {
216	+ $api = SitesManagerApi::getInstance();
217	+ $allSitesId = $api->getAllSitesId();
218	+ } catch (\Exception $e) {
219	+ $allSitesId = array();
220	+ }
221	+ $this->idsitesByAccess['superuser'] = $allSitesId;
222	+ }
223	+ } elseif (isset($this->login)) {
224	+ if (empty($this->idsitesByAccess['view'])
225	+ && empty($this->idsitesByAccess['write'])
226	+ && empty($this->idsitesByAccess['admin'])
227	+ ) {
228	+ // we join with site in case there are rows in access for an idsite that doesn't exist anymore
229	+ // (backward compatibility ; before we deleted the site without deleting rows in _access table)
230	+ $accessRaw = $this->getRawSitesWithSomeViewAccess($this->login);
231	+
232	+ foreach ($accessRaw as $access) {
233	+ $accessType = $access['access'];
234	+ $this->idsitesByAccess[$accessType][] = $access['idsite'];
235	+
236	+ if ($this->roleProvider->isValidRole($accessType)) {
237	+ foreach ($this->capabilityProvider->getAllCapabilities() as $capability) {
238	+ if ($capability->hasRoleCapability($accessType)) {
239	+ // we automatically add this capability
240	+ if (!isset($this->idsitesByAccess[$capability->getId()])) {
241	+ $this->idsitesByAccess[$capability->getId()] = array();
242	+ }
243	+ $this->idsitesByAccess[$capability->getId()][] = $access['idsite'];
244	+ }
245	+ }
246	+ }
247	+ }
248	+
249	+ /**
250	+ * Triggered after the initial access levels and permissions for the current user are loaded. Use this
251	+ * event to modify the current user's permissions (for example, making sure every user has view access
252	+ * to a specific site).
253	+ *
254	+ * Example
255	+ *
256	+ * function (&$idsitesByAccess, $login) {
257	+ * if ($login == 'somespecialuser') {
258	+ * return;
259	+ * }
260	+ *
261	+ * $idsitesByAccess['view'][] = $mySpecialIdSite;
262	+ * }
263	+ *
264	+ * @param array[] &$idsitesByAccess The current user's access levels for individual sites. Maps role and
265	+ * capability IDs to list of site IDs, eg:
266	+ *
267	+ * ```
268	+ * [
269	+ * 'view' => [1, 2, 3],
270	+ * 'write' => [4, 5],
271	+ * 'admin' => [],
272	+ * ]
273	+ * ```
274	+ * @param string $login The current user's login.
275	+ */
276	+ Piwik::postEvent('Access.modifyUserAccess', [&$this->idsitesByAccess, $this->login]);
277	+ }
278	+ }
279	+ }
280	+
281	+ /**
282	+ * We bypass the normal auth method and give the current user Super User rights.
283	+ * This should be very carefully used.
284	+ *
285	+ * @param bool $bool
286	+ */
287	+ public function setSuperUserAccess($bool = true)
288	+ {
289	+ $this->hasSuperUserAccess = (bool) $bool;
290	+
291	+ if ($bool) {
292	+ $this->makeSureLoginNameIsSet();
293	+ } else {
294	+ $this->resetSites();
295	+ }
296	+ }
297	+
298	+ /**
299	+ * Returns true if the current user is logged in as the Super User
300	+ *
301	+ * @return bool
302	+ */
303	+ public function hasSuperUserAccess()
304	+ {
305	+ return $this->hasSuperUserAccess;
306	+ }
307	+
308	+ /**
309	+ * Returns the current user login
310	+ *
311	+ * @return string\|null
312	+ */
313	+ public function getLogin()
314	+ {
315	+ return $this->login;
316	+ }
317	+
318	+ /**
319	+ * Returns the token_auth used to authenticate this user in the API
320	+ *
321	+ * @return string\|null
322	+ */
323	+ public function getTokenAuth()
324	+ {
325	+ return $this->token_auth;
326	+ }
327	+
328	+ /**
329	+ * Returns an array of ID sites for which the user has at least a VIEW access.
330	+ * Which means VIEW OR WRITE or ADMIN or SUPERUSER.
331	+ *
332	+ * @return array Example if the user is ADMIN for 4
333	+ * and has VIEW access for 1 and 7, it returns array(1, 4, 7);
334	+ */
335	+ public function getSitesIdWithAtLeastViewAccess()
336	+ {
337	+ $this->loadSitesIfNeeded();
338	+
339	+ return array_unique(array_merge(
340	+ $this->idsitesByAccess['view'],
341	+ $this->idsitesByAccess['write'],
342	+ $this->idsitesByAccess['admin'],
343	+ $this->idsitesByAccess['superuser'])
344	+ );
345	+ }
346	+
347	+ /**
348	+ * Returns an array of ID sites for which the user has at least a WRITE access.
349	+ * Which means WRITE or ADMIN or SUPERUSER.
350	+ *
351	+ * @return array Example if the user is WRITE for 4 and 8
352	+ * and has VIEW access for 1 and 7, it returns array(4, 8);
353	+ */
354	+ public function getSitesIdWithAtLeastWriteAccess()
355	+ {
356	+ $this->loadSitesIfNeeded();
357	+
358	+ return array_unique(array_merge(
359	+ $this->idsitesByAccess['write'],
360	+ $this->idsitesByAccess['admin'],
361	+ $this->idsitesByAccess['superuser'])
362	+ );
363	+ }
364	+
365	+ /**
366	+ * Returns an array of ID sites for which the user has an ADMIN access.
367	+ *
368	+ * @return array Example if the user is ADMIN for 4 and 8
369	+ * and has VIEW access for 1 and 7, it returns array(4, 8);
370	+ */
371	+ public function getSitesIdWithAdminAccess()
372	+ {
373	+ $this->loadSitesIfNeeded();
374	+
375	+ return array_unique(array_merge(
376	+ $this->idsitesByAccess['admin'],
377	+ $this->idsitesByAccess['superuser'])
378	+ );
379	+ }
380	+
381	+ /**
382	+ * Returns an array of ID sites for which the user has a VIEW access only.
383	+ *
384	+ * @return array Example if the user is ADMIN for 4
385	+ * and has VIEW access for 1 and 7, it returns array(1, 7);
386	+ * @see getSitesIdWithAtLeastViewAccess()
387	+ */
388	+ public function getSitesIdWithViewAccess()
389	+ {
390	+ $this->loadSitesIfNeeded();
391	+
392	+ return $this->idsitesByAccess['view'];
393	+ }
394	+
395	+ /**
396	+ * Returns an array of ID sites for which the user has a WRITE access only.
397	+ *
398	+ * @return array Example if the user is ADMIN for 4
399	+ * and has WRITE access for 1 and 7, it returns array(1, 7);
400	+ * @see getSitesIdWithAtLeastWriteAccess()
401	+ */
402	+ public function getSitesIdWithWriteAccess()
403	+ {
404	+ $this->loadSitesIfNeeded();
405	+
406	+ return $this->idsitesByAccess['write'];
407	+ }
408	+
409	+ /**
410	+ * Throws an exception if the user is not the SuperUser
411	+ *
412	+ * @throws \Piwik\NoAccessException
413	+ */
414	+ public function checkUserHasSuperUserAccess()
415	+ {
416	+ if (!$this->hasSuperUserAccess()) {
417	+ $this->throwNoAccessException(Piwik::translate('General_ExceptionPrivilege', array("'superuser'")));
418	+ }
419	+ }
420	+
421	+ /**
422	+ * Returns `true` if the current user has admin access to at least one site.
423	+ *
424	+ * @return bool
425	+ */
426	+ public function isUserHasSomeWriteAccess()
427	+ {
428	+ if ($this->hasSuperUserAccess()) {
429	+ return true;
430	+ }
431	+
432	+ $idSitesAccessible = $this->getSitesIdWithAtLeastWriteAccess();
433	+
434	+ return count($idSitesAccessible) > 0;
435	+ }
436	+
437	+ /**
438	+ * Returns `true` if the current user has admin access to at least one site.
439	+ *
440	+ * @return bool
441	+ */
442	+ public function isUserHasSomeAdminAccess()
443	+ {
444	+ if ($this->hasSuperUserAccess()) {
445	+ return true;
446	+ }
447	+
448	+ $idSitesAccessible = $this->getSitesIdWithAdminAccess();
449	+
450	+ return count($idSitesAccessible) > 0;
451	+ }
452	+
453	+ /**
454	+ * If the user doesn't have an WRITE access for at least one website, throws an exception
455	+ *
456	+ * @throws \Piwik\NoAccessException
457	+ */
458	+ public function checkUserHasSomeWriteAccess()
459	+ {
460	+ if (!$this->isUserHasSomeWriteAccess()) {
461	+ $this->throwNoAccessException(Piwik::translate('General_ExceptionPrivilegeAtLeastOneWebsite', array('write')));
462	+ }
463	+ }
464	+
465	+ /**
466	+ * If the user doesn't have an ADMIN access for at least one website, throws an exception
467	+ *
468	+ * @throws \Piwik\NoAccessException
469	+ */
470	+ public function checkUserHasSomeAdminAccess()
471	+ {
472	+ if (!$this->isUserHasSomeAdminAccess()) {
473	+ $this->throwNoAccessException(Piwik::translate('General_ExceptionPrivilegeAtLeastOneWebsite', array('admin')));
474	+ }
475	+ }
476	+
477	+ /**
478	+ * If the user doesn't have any view permission, throw exception
479	+ *
480	+ * @throws \Piwik\NoAccessException
481	+ */
482	+ public function checkUserHasSomeViewAccess()
483	+ {
484	+ if ($this->hasSuperUserAccess()) {
485	+ return;
486	+ }
487	+
488	+ $idSitesAccessible = $this->getSitesIdWithAtLeastViewAccess();
489	+
490	+ if (count($idSitesAccessible) == 0) {
491	+ $this->throwNoAccessException(Piwik::translate('General_ExceptionPrivilegeAtLeastOneWebsite', array('view')));
492	+ }
493	+ }
494	+
495	+ /**
496	+ * This method checks that the user has ADMIN access for the given list of websites.
497	+ * If the user doesn't have ADMIN access for at least one website of the list, we throw an exception.
498	+ *
499	+ * @param int\|array $idSites List of ID sites to check
500	+ * @throws \Piwik\NoAccessException If for any of the websites the user doesn't have an ADMIN access
501	+ */
502	+ public function checkUserHasAdminAccess($idSites)
503	+ {
504	+ if ($this->hasSuperUserAccess()) {
505	+ return;
506	+ }
507	+
508	+ $idSites = $this->getIdSites($idSites);
509	+ $idSitesAccessible = $this->getSitesIdWithAdminAccess();
510	+
511	+ foreach ($idSites as $idsite) {
512	+ if (!in_array($idsite, $idSitesAccessible)) {
513	+ $this->throwNoAccessException(Piwik::translate('General_ExceptionPrivilegeAccessWebsite', array("'admin'", $idsite)));
514	+ }
515	+ }
516	+ }
517	+
518	+ /**
519	+ * This method checks that the user has VIEW or ADMIN access for the given list of websites.
520	+ * If the user doesn't have VIEW or ADMIN access for at least one website of the list, we throw an exception.
521	+ *
522	+ * @param int\|array\|string $idSites List of ID sites to check (integer, array of integers, string comma separated list of integers)
523	+ * @throws \Piwik\NoAccessException If for any of the websites the user doesn't have an VIEW or ADMIN access
524	+ */
525	+ public function checkUserHasViewAccess($idSites)
526	+ {
527	+ if ($this->hasSuperUserAccess()) {
528	+ return;
529	+ }
530	+
531	+ $idSites = $this->getIdSites($idSites);
532	+ $idSitesAccessible = $this->getSitesIdWithAtLeastViewAccess();
533	+
534	+ foreach ($idSites as $idsite) {
535	+ if (!in_array($idsite, $idSitesAccessible)) {
536	+ $this->throwNoAccessException(Piwik::translate('General_ExceptionPrivilegeAccessWebsite', array("'view'", $idsite)));
537	+ }
538	+ }
539	+ }
540	+
541	+ /**
542	+ * This method checks that the user has VIEW or ADMIN access for the given list of websites.
543	+ * If the user doesn't have VIEW or ADMIN access for at least one website of the list, we throw an exception.
544	+ *
545	+ * @param int\|array\|string $idSites List of ID sites to check (integer, array of integers, string comma separated list of integers)
546	+ * @throws \Piwik\NoAccessException If for any of the websites the user doesn't have an VIEW or ADMIN access
547	+ */
548	+ public function checkUserHasWriteAccess($idSites)
549	+ {
550	+ if ($this->hasSuperUserAccess()) {
551	+ return;
552	+ }
553	+
554	+ $idSites = $this->getIdSites($idSites);
555	+ $idSitesAccessible = $this->getSitesIdWithAtLeastWriteAccess();
556	+
557	+ foreach ($idSites as $idsite) {
558	+ if (!in_array($idsite, $idSitesAccessible)) {
559	+ $this->throwNoAccessException(Piwik::translate('General_ExceptionPrivilegeAccessWebsite', array("'write'", $idsite)));
560	+ }
561	+ }
562	+ }
563	+
564	+ public function checkUserIsNotAnonymous()
565	+ {
566	+ if ($this->hasSuperUserAccess()) {
567	+ return;
568	+ }
569	+ if (Piwik::isUserIsAnonymous()) {
570	+ $this->throwNoAccessException(Piwik::translate('General_YouMustBeLoggedIn'));
571	+ }
572	+ }
573	+
574	+ private function getSitesIdWithCapability($capability)
575	+ {
576	+ if (!empty($this->idsitesByAccess[$capability])) {
577	+ return $this->idsitesByAccess[$capability];
578	+ }
579	+ return array();
580	+ }
581	+
582	+ public function checkUserHasCapability($idSites, $capability)
583	+ {
584	+ if ($this->hasSuperUserAccess()) {
585	+ return;
586	+ }
587	+
588	+ $idSites = $this->getIdSites($idSites);
589	+ $idSitesAccessible = $this->getSitesIdWithCapability($capability);
590	+
591	+ foreach ($idSites as $idsite) {
592	+ if (!in_array($idsite, $idSitesAccessible)) {
593	+ $this->throwNoAccessException(Piwik::translate('ExceptionCapabilityAccessWebsite', array("'" . $capability ."'", $idsite)));
594	+ }
595	+ }
596	+
597	+ // a capability applies only when the user also has at least view access
598	+ $this->checkUserHasViewAccess($idSites);
599	+ }
600	+
601	+ /**
602	+ * @param int\|array\|string $idSites
603	+ * @return array
604	+ * @throws \Piwik\NoAccessException
605	+ */
606	+ protected function getIdSites($idSites)
607	+ {
608	+ if ($idSites === 'all') {
609	+ $idSites = $this->getSitesIdWithAtLeastViewAccess();
610	+ }
611	+
612	+ $idSites = Site::getIdSitesFromIdSitesString($idSites);
613	+
614	+ if (empty($idSites)) {
615	+ $this->throwNoAccessException("The parameter 'idSite=' is missing from the request.");
616	+ }
617	+
618	+ return $idSites;
619	+ }
620	+
621	+ /**
622	+ * Executes a callback with superuser privileges, making sure those privileges are rescinded
623	+ * before this method exits. Privileges will be rescinded even if an exception is thrown.
624	+ *
625	+ * @param callback $function The callback to execute. Should accept no arguments.
626	+ * @return mixed The result of `$function`.
627	+ * @throws Exception rethrows any exceptions thrown by `$function`.
628	+ * @api
629	+ */
630	+ public static function doAsSuperUser($function)
631	+ {
632	+ $isSuperUser = self::getInstance()->hasSuperUserAccess();
633	+
634	+ if ($isSuperUser) {
635	+ return $function();
636	+ }
637	+
638	+ $access = self::getInstance();
639	+ $login = $access->getLogin();
640	+ $shouldResetLogin = empty($login); // make sure to reset login if a login was set by "makeSureLoginNameIsSet()"
641	+ $access->setSuperUserAccess(true);
642	+
643	+ try {
644	+ $result = $function();
645	+ } catch (Exception $ex) {
646	+ $access->setSuperUserAccess($isSuperUser);
647	+ if ($shouldResetLogin) {
648	+ $access->login = null;
649	+ }
650	+
651	+ throw $ex;
652	+ }
653	+
654	+ if ($shouldResetLogin) {
655	+ $access->login = null;
656	+ }
657	+ $access->setSuperUserAccess($isSuperUser);
658	+
659	+ return $result;
660	+ }
661	+
662	+ /**
663	+ * Returns the level of access the current user has to the given site.
664	+ *
665	+ * @param int $idSite The site to check.
666	+ * @return string The access level, eg, 'view', 'admin', 'noaccess'.
667	+ */
668	+ public function getRoleForSite($idSite)
669	+ {
670	+ if ($this->hasSuperUserAccess
671	+ \|\| in_array($idSite, $this->getSitesIdWithAdminAccess())
672	+ ) {
673	+ return 'admin';
674	+ }
675	+
676	+ if (in_array($idSite, $this->getSitesIdWithWriteAccess())) {
677	+ return 'write';
678	+ }
679	+
680	+ if (in_array($idSite, $this->getSitesIdWithViewAccess())) {
681	+ return 'view';
682	+ }
683	+
684	+ return 'noaccess';
685	+ }
686	+
687	+ /**
688	+ * Returns the capabilities the current user has for a given site.
689	+ *
690	+ * @param int $idSite The site to check.
691	+ * @return string[] The capabilities the user has.
692	+ */
693	+ public function getCapabilitiesForSite($idSite)
694	+ {
695	+ $result = [];
696	+ foreach ($this->capabilityProvider->getAllCapabilityIds() as $capabilityId) {
697	+ if (empty($this->idsitesByAccess[$capabilityId])) {
698	+ continue;
699	+ }
700	+
701	+ if (in_array($idSite, $this->idsitesByAccess[$capabilityId])) {
702	+ $result[] = $capabilityId;
703	+ }
704	+ }
705	+ return $result;
706	+ }
707	+
708	+ /**
709	+ * Throw a NoAccessException with the given message, or a more generic 'You need to log in' message if the
710	+ * user is not currently logged in (e.g. if session has expired).
711	+ * @param $message
712	+ * @throws NoAccessException
713	+ */
714	+ private function throwNoAccessException($message)
715	+ {
716	+ if (Piwik::isUserIsAnonymous() && !Request::isRootRequestApiRequest()) {
717	+ $message = Piwik::translate('General_YouMustBeLoggedIn');
718	+ }
719	+ // Try to detect whether user was previously logged in so that we can display a different message
720	+ $referrer = Url::getReferrer();
721	+ $matomoUrl = SettingsPiwik::getPiwikUrl();
722	+ if ($referrer && $matomoUrl && Url::isValidHost(Url::getHostFromUrl($referrer)) &&
723	+ strpos($referrer, $matomoUrl) === 0
724	+ ) {
725	+ $message = Piwik::translate('General_YourSessionHasExpired');
726	+ }
727	+
728	+ throw new NoAccessException($message);
729	+ }
730	+ }
731	+
732	+ /**
733	+ * Exception thrown when a user doesn't have sufficient access to a resource.
734	+ *
735	+ * @api
736	+ */
737	+ class NoAccessException extends InvalidRequestParameterException
738	+ {
739	+ }

app/core/Access/CapabilitiesProvider.php ADDED Viewed

	@@ -0,0 +1,123 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik\Access;
10	+
11	+ use Exception;
12	+ use Piwik\CacheId;
13	+ use Piwik\Piwik;
14	+ use Piwik\Cache as PiwikCache;
15	+
16	+ class CapabilitiesProvider
17	+ {
18	+ /**
19	+ * @return Capability[]
20	+ */
21	+ public function getAllCapabilities()
22	+ {
23	+ $cacheId = CacheId::siteAware(CacheId::languageAware('Capabilities'));
24	+ $cache = PiwikCache::getTransientCache();
25	+
26	+ if (!$cache->contains($cacheId)) {
27	+ $capabilities = array();
28	+
29	+ /**
30	+ * Triggered to add new capabilities.
31	+ *
32	+ * Example
33	+ *
34	+ * public function addCapabilities(&$capabilities)
35	+ * {
36	+ * $capabilities[] = new MyNewCapabilitiy();
37	+ * }
38	+ *
39	+ * @param Capability[] $reports An array of reports
40	+ * @internal
41	+ */
42	+ Piwik::postEvent('Access.Capability.addCapabilities', array(&$capabilities));
43	+
44	+ /**
45	+ * Triggered to filter / restrict capabilities.
46	+ *
47	+ * Example
48	+ *
49	+ * public function filterCapabilities(&$capabilities)
50	+ * {
51	+ * foreach ($capabilities as $index => $capability) {
52	+ * if ($capability->getId() === 'tagmanager_write') {}
53	+ * unset($capabilities[$index]); // remove the given capability
54	+ * }
55	+ * }
56	+ * }
57	+ *
58	+ * @param Capability[] $reports An array of reports
59	+ * @internal
60	+ */
61	+ Piwik::postEvent('Access.Capability.filterCapabilities', array(&$capabilities));
62	+
63	+ $capabilities = array_values($capabilities);
64	+
65	+ $this->checkCapabilityIds($capabilities);
66	+
67	+ $cache->save($cacheId, $capabilities);
68	+ return $capabilities;
69	+ }
70	+
71	+ return $cache->fetch($cacheId);
72	+ }
73	+
74	+ /**
75	+ * @param $capabilityId
76	+ * @return Capability\|null
77	+ */
78	+ public function getCapability($capabilityId)
79	+ {
80	+ foreach ($this->getAllCapabilities() as $capability) {
81	+ if ($capabilityId === $capability->getId()) {
82	+ return $capability;
83	+ }
84	+ }
85	+ }
86	+
87	+ public function getAllCapabilityIds()
88	+ {
89	+ $ids = array();
90	+ foreach ($this->getAllCapabilities() as $capability) {
91	+ $ids[] = $capability->getId();
92	+ }
93	+ return $ids;
94	+ }
95	+
96	+ public function isValidCapability($capabilityId)
97	+ {
98	+ $capabilities = $this->getAllCapabilityIds();
99	+
100	+ return in_array($capabilityId, $capabilities, true);
101	+ }
102	+
103	+ public function checkValidCapability($capabilityId)
104	+ {
105	+ if (!$this->isValidCapability($capabilityId)) {
106	+ $capabilities = $this->getAllCapabilityIds();
107	+ throw new Exception(Piwik::translate("UsersManager_ExceptionAccessValues", implode(", ", $capabilities)));
108	+ }
109	+ }
110	+
111	+ /**
112	+ * @param Capability[] $capabilities
113	+ */
114	+ private function checkCapabilityIds($capabilities)
115	+ {
116	+ foreach ($capabilities as $capability) {
117	+ $id = $capability->getId();
118	+ if (preg_match('/[^a-zA-Z0-9_-]/', $id)) {
119	+ throw new \Exception("Capability with invalid ID found: '$id'. Valid characters are 'a-zA-Z0-9_-'.");
120	+ }
121	+ }
122	+ }
123	+ }

app/core/Access/Capability.php ADDED Viewed

	@@ -0,0 +1,29 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik\Access;
10	+
11	+ abstract class Capability
12	+ {
13	+ abstract public function getId();
14	+ abstract public function getName();
15	+ abstract public function getCategory();
16	+ abstract public function getDescription();
17	+ abstract public function getIncludedInRoles();
18	+
19	+ public function getHelpUrl()
20	+ {
21	+ return '';
22	+ }
23	+
24	+ public function hasRoleCapability($idRole)
25	+ {
26	+ return in_array($idRole, $this->getIncludedInRoles(), true);
27	+ }
28	+
29	+ }

app/core/Access/Role.php ADDED Viewed

	@@ -0,0 +1,22 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik\Access;
10	+
11	+ abstract class Role
12	+ {
13	+ abstract public function getName();
14	+ abstract public function getId();
15	+ abstract public function getDescription();
16	+
17	+ public function getHelpUrl()
18	+ {
19	+ return '';
20	+ }
21	+
22	+ }

app/core/Access/Role/Admin.php ADDED Viewed

	@@ -0,0 +1,40 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik\Access\Role;
10	+
11	+ use Piwik\Access\Role;
12	+ use Piwik\Piwik;
13	+
14	+ class Admin extends Role
15	+ {
16	+ const ID = 'admin';
17	+
18	+ public function getName()
19	+ {
20	+ return Piwik::translate('UsersManager_PrivAdmin');
21	+ }
22	+
23	+ public function getId()
24	+ {
25	+ return self::ID;
26	+ }
27	+
28	+ public function getDescription()
29	+ {
30	+ return Piwik::translate('UsersManager_PrivAdminDescription', array(
31	+ Piwik::translate('UsersManager_PrivWrite')
32	+ ));
33	+ }
34	+
35	+ public function getHelpUrl()
36	+ {
37	+ return 'https://matomo.org/faq/general/faq_69/';
38	+ }
39	+
40	+ }

app/core/Access/Role/View.php ADDED Viewed

	@@ -0,0 +1,39 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik\Access\Role;
10	+
11	+ use Piwik\Access\Role;
12	+ use Piwik\Piwik;
13	+
14	+ class View extends Role
15	+ {
16	+ const ID = 'view';
17	+
18	+ public function getName()
19	+ {
20	+ return Piwik::translate('UsersManager_PrivView');
21	+ }
22	+
23	+ public function getId()
24	+ {
25	+ return self::ID;
26	+ }
27	+
28	+ public function getDescription()
29	+ {
30	+ return Piwik::translate('UsersManager_PrivViewDescription');
31	+ }
32	+
33	+ public function getHelpUrl()
34	+ {
35	+ return 'https://matomo.org/faq/general/faq_70/';
36	+ }
37	+
38	+
39	+ }

app/core/Access/Role/Write.php ADDED Viewed

	@@ -0,0 +1,38 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik\Access\Role;
10	+
11	+ use Piwik\Access\Role;
12	+ use Piwik\Piwik;
13	+
14	+ class Write extends Role
15	+ {
16	+ const ID = 'write';
17	+
18	+ public function getName()
19	+ {
20	+ return Piwik::translate('UsersManager_PrivWrite');
21	+ }
22	+
23	+ public function getId()
24	+ {
25	+ return self::ID;
26	+ }
27	+
28	+ public function getDescription()
29	+ {
30	+ return Piwik::translate('UsersManager_PrivWriteDescription');
31	+ }
32	+
33	+ public function getHelpUrl()
34	+ {
35	+ return '';
36	+ }
37	+
38	+ }

app/core/Access/RolesProvider.php ADDED Viewed

	@@ -0,0 +1,62 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik\Access;
10	+
11	+ use Piwik\Access\Role\Admin;
12	+ use Piwik\Access\Role\View;
13	+ use Piwik\Access\Role\Write;
14	+ use Piwik\Piwik;
15	+ use Exception;
16	+
17	+ class RolesProvider
18	+ {
19	+ /**
20	+ * @return Role[]
21	+ */
22	+ public function getAllRoles()
23	+ {
24	+ return array(
25	+ new View(),
26	+ new Write(),
27	+ new Admin()
28	+ );
29	+ }
30	+
31	+ /**
32	+ * Returns the list of the existing Access level.
33	+ * Useful when a given API method requests a given acccess Level.
34	+ * We first check that the required access level exists.
35	+ *
36	+ * @return array
37	+ */
38	+ public function getAllRoleIds()
39	+ {
40	+ $ids = array();
41	+ foreach ($this->getAllRoles() as $role) {
42	+ $ids[] = $role->getId();
43	+ }
44	+ return $ids;
45	+ }
46	+
47	+ public function isValidRole($roleId)
48	+ {
49	+ $roles = $this->getAllRoleIds();
50	+
51	+ return in_array($roleId, $roles, true);
52	+ }
53	+
54	+ public function checkValidRole($roleId)
55	+ {
56	+ if (!$this->isValidRole($roleId)) {
57	+ $roles = $this->getAllRoleIds();
58	+ throw new Exception(Piwik::translate("UsersManager_ExceptionAccessValues", implode(", ", $roles)));
59	+ }
60	+ }
61	+
62	+ }

app/core/Application/Environment.php ADDED Viewed

	@@ -0,0 +1,252 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ */
8	+
9	+ namespace Piwik\Application;
10	+
11	+ use DI\Container;
12	+ use Piwik\Application\Kernel\EnvironmentValidator;
13	+ use Piwik\Application\Kernel\GlobalSettingsProvider;
14	+ use Piwik\Application\Kernel\PluginList;
15	+ use Piwik\Container\ContainerFactory;
16	+ use Piwik\Container\StaticContainer;
17	+ use Piwik\Piwik;
18	+
19	+ /**
20	+ * Encapsulates Piwik environment setup and access.
21	+ *
22	+ * The Piwik environment consists of two main parts: the kernel and the DI container.
23	+ *
24	+ * The 'kernel' is the core part of Piwik that cannot be modified / extended through the DI container.
25	+ * It includes components that are required to create the DI container.
26	+ *
27	+ * Currently the only objects in the 'kernel' are a GlobalSettingsProvider object and a
28	+ * PluginList object. The GlobalSettingsProvider object is required for the current PluginList
29	+ * implementation and for checking whether Development mode is enabled. The PluginList is
30	+ * needed in order to determine what plugins are activated, since plugins can provide their
31	+ * own DI configuration.
32	+ *
33	+ * The DI container contains every other Piwik object, including the Plugin\Manager,
34	+ * plugin API instances, dependent services, etc. Plugins and users can override/extend
35	+ * the objects in this container.
36	+ *
37	+ * NOTE: DI support in Piwik is currently a work in process; not everything is currently
38	+ * stored in the DI container, but we are working towards this.
39	+ */
40	+ class Environment
41	+ {
42	+ /**
43	+ * @internal
44	+ * @var EnvironmentManipulator
45	+ */
46	+ private static $globalEnvironmentManipulator = null;
47	+
48	+ /**
49	+ * @var string
50	+ */
51	+ private $environment;
52	+
53	+ /**
54	+ * @var array
55	+ */
56	+ private $definitions;
57	+
58	+ /**
59	+ * @var Container
60	+ */
61	+ private $container;
62	+
63	+ /**
64	+ * @var GlobalSettingsProvider
65	+ */
66	+ private $globalSettingsProvider;
67	+
68	+ /**
69	+ * @var PluginList
70	+ */
71	+ private $pluginList;
72	+
73	+ /**
74	+ * @param string $environment
75	+ * @param array $definitions
76	+ */
77	+ public function __construct($environment, array $definitions = array())
78	+ {
79	+ $this->environment = $environment;
80	+ $this->definitions = $definitions;
81	+ }
82	+
83	+ public function getEnvironmentName()
84	+ {
85	+ return $this->environment;
86	+ }
87	+
88	+ /**
89	+ * Initializes the kernel globals and DI container.
90	+ */
91	+ public function init()
92	+ {
93	+ $this->invokeBeforeContainerCreatedHook();
94	+
95	+ $this->container = $this->createContainer();
96	+ $this->container->set(self::class, $this);
97	+
98	+ StaticContainer::push($this->container);
99	+
100	+ $this->validateEnvironment();
101	+
102	+ $this->invokeEnvironmentBootstrappedHook();
103	+
104	+ Piwik::postEvent('Environment.bootstrapped'); // this event should be removed eventually
105	+ }
106	+
107	+ /**
108	+ * Destroys an environment. MUST be called when embedding environments.
109	+ */
110	+ public function destroy()
111	+ {
112	+ StaticContainer::pop();
113	+ }
114	+
115	+ /**
116	+ * Returns the DI container. All Piwik objects for a specific Piwik instance should be stored
117	+ * in this container.
118	+ *
119	+ * @return Container
120	+ */
121	+ public function getContainer()
122	+ {
123	+ return $this->container;
124	+ }
125	+
126	+ /**
127	+ * @link http://php-di.org/doc/container-configuration.html
128	+ */
129	+ private function createContainer()
130	+ {
131	+ $pluginList = $this->getPluginListCached();
132	+ $settings = $this->getGlobalSettingsCached();
133	+
134	+ $extraDefinitions = $this->getExtraDefinitionsFromManipulators();
135	+ $definitions = array_merge(StaticContainer::getDefinitions(), $extraDefinitions, array($this->definitions));
136	+
137	+ $environments = array($this->environment);
138	+ $environments = array_merge($environments, $this->getExtraEnvironmentsFromManipulators());
139	+
140	+ $containerFactory = new ContainerFactory($pluginList, $settings, $environments, $definitions);
141	+ return $containerFactory->create();
142	+ }
143	+
144	+ protected function getGlobalSettingsCached()
145	+ {
146	+ if ($this->globalSettingsProvider === null) {
147	+ $original = $this->getGlobalSettings();
148	+ $globalSettingsProvider = $this->getGlobalSettingsProviderOverride($original);
149	+
150	+ $this->globalSettingsProvider = $globalSettingsProvider ?: $original;
151	+ }
152	+ return $this->globalSettingsProvider;
153	+ }
154	+
155	+ protected function getPluginListCached()
156	+ {
157	+ if ($this->pluginList === null) {
158	+ $pluginList = $this->getPluginListOverride();
159	+ $this->pluginList = $pluginList ?: $this->getPluginList();
160	+ }
161	+ return $this->pluginList;
162	+ }
163	+
164	+ /**
165	+ * Returns the kernel global GlobalSettingsProvider object. Derived classes can override this method
166	+ * to provide a different implementation.
167	+ *
168	+ * @return null\|GlobalSettingsProvider
169	+ */
170	+ protected function getGlobalSettings()
171	+ {
172	+ return new GlobalSettingsProvider();
173	+ }
174	+
175	+ /**
176	+ * Returns the kernel global PluginList object. Derived classes can override this method to
177	+ * provide a different implementation.
178	+ *
179	+ * @return PluginList
180	+ */
181	+ protected function getPluginList()
182	+ {
183	+ // TODO: in tracker should only load tracker plugins. can't do properly until tracker entrypoint is encapsulated.
184	+ return new PluginList($this->getGlobalSettingsCached());
185	+ }
186	+
187	+ private function validateEnvironment()
188	+ {
189	+ /** @var EnvironmentValidator $validator */
190	+ $validator = $this->container->get('Piwik\Application\Kernel\EnvironmentValidator');
191	+ $validator->validate();
192	+ }
193	+
194	+ /**
195	+ * @param EnvironmentManipulator $manipulator
196	+ * @internal
197	+ */
198	+ public static function setGlobalEnvironmentManipulator(EnvironmentManipulator $manipulator)
199	+ {
200	+ self::$globalEnvironmentManipulator = $manipulator;
201	+ }
202	+
203	+ private function getGlobalSettingsProviderOverride(GlobalSettingsProvider $original)
204	+ {
205	+ if (self::$globalEnvironmentManipulator) {
206	+ return self::$globalEnvironmentManipulator->makeGlobalSettingsProvider($original);
207	+ } else {
208	+ return null;
209	+ }
210	+ }
211	+
212	+ private function invokeBeforeContainerCreatedHook()
213	+ {
214	+ if (self::$globalEnvironmentManipulator) {
215	+ return self::$globalEnvironmentManipulator->beforeContainerCreated();
216	+ }
217	+ }
218	+
219	+ private function getExtraDefinitionsFromManipulators()
220	+ {
221	+ if (self::$globalEnvironmentManipulator) {
222	+ return self::$globalEnvironmentManipulator->getExtraDefinitions();
223	+ } else {
224	+ return array();
225	+ }
226	+ }
227	+
228	+ private function invokeEnvironmentBootstrappedHook()
229	+ {
230	+ if (self::$globalEnvironmentManipulator) {
231	+ self::$globalEnvironmentManipulator->onEnvironmentBootstrapped();
232	+ }
233	+ }
234	+
235	+ private function getExtraEnvironmentsFromManipulators()
236	+ {
237	+ if (self::$globalEnvironmentManipulator) {
238	+ return self::$globalEnvironmentManipulator->getExtraEnvironments();
239	+ } else {
240	+ return array();
241	+ }
242	+ }
243	+
244	+ private function getPluginListOverride()
245	+ {
246	+ if (self::$globalEnvironmentManipulator) {
247	+ return self::$globalEnvironmentManipulator->makePluginList($this->getGlobalSettingsCached());
248	+ } else {
249	+ return null;
250	+ }
251	+ }
252	+ }

app/core/Application/EnvironmentManipulator.php ADDED Viewed

	@@ -0,0 +1,59 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ */
8	+
9	+ namespace Piwik\Application;
10	+
11	+ use Piwik\Application\Kernel\GlobalSettingsProvider;
12	+ use Piwik\Application\Kernel\PluginList;
13	+
14	+ /**
15	+ * Used to manipulate Environment instances before the container is created.
16	+ * Only used by the testing environment setup code, shouldn't be used anywhere
17	+ * else.
18	+ */
19	+ interface EnvironmentManipulator
20	+ {
21	+ /**
22	+ * Create a custom GlobalSettingsProvider kernel object, overriding the default behavior.
23	+ *
24	+ * @return GlobalSettingsProvider
25	+ */
26	+ public function makeGlobalSettingsProvider(GlobalSettingsProvider $original);
27	+
28	+ /**
29	+ * Create a custom PluginList kernel object, overriding the default behavior.@deprecated
30	+ *
31	+ * @param GlobalSettingsProvider $globalSettingsProvider
32	+ * @return PluginList
33	+ */
34	+ public function makePluginList(GlobalSettingsProvider $globalSettingsProvider);
35	+
36	+ /**
37	+ * Invoked before the container is created.
38	+ */
39	+ public function beforeContainerCreated();
40	+
41	+ /**
42	+ * Return an array of definition arrays that override DI config specified in PHP config files.
43	+ *
44	+ * @return array[]
45	+ */
46	+ public function getExtraDefinitions();
47	+
48	+ /**
49	+ * Invoked after the container is created and the environment is considered bootstrapped.
50	+ */
51	+ public function onEnvironmentBootstrapped();
52	+
53	+ /**
54	+ * Return an array of environment names to apply after the normal environment.
55	+ *
56	+ * @return string[]
57	+ */
58	+ public function getExtraEnvironments();
59	+ }

app/core/Application/Kernel/EnvironmentValidator.php ADDED Viewed

	@@ -0,0 +1,153 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ */
8	+
9	+ namespace Piwik\Application\Kernel;
10	+
11	+ use Piwik\Common;
12	+ use Piwik\Config;
13	+ use Piwik\Exception\InvalidRequestParameterException;
14	+ use Piwik\Exception\NotYetInstalledException;
15	+ use Piwik\Filechecks;
16	+ use Piwik\Piwik;
17	+ use Piwik\SettingsPiwik;
18	+ use Piwik\SettingsServer;
19	+ use Piwik\Translation\Translator;
20	+
21	+ /**
22	+ * Validates the Piwik environment. This includes making sure the required config files
23	+ * are present, and triggering the correct behaviour if otherwise.
24	+ */
25	+ class EnvironmentValidator
26	+ {
27	+ /**
28	+ * @var GlobalSettingsProvider
29	+ */
30	+ protected $settingsProvider;
31	+
32	+ /**
33	+ * @var Translator
34	+ */
35	+ protected $translator;
36	+
37	+ public function __construct(GlobalSettingsProvider $settingsProvider, Translator $translator)
38	+ {
39	+ $this->settingsProvider = $settingsProvider;
40	+ $this->translator = $translator;
41	+ }
42	+
43	+ public function validate()
44	+ {
45	+ $this->checkConfigFileExists($this->settingsProvider->getPathGlobal());
46	+
47	+ if(SettingsPiwik::isPiwikInstalled()) {
48	+ $this->checkConfigFileExists($this->settingsProvider->getPathLocal(), $startInstaller = false);
49	+ return;
50	+ }
51	+
52	+ $startInstaller = true;
53	+
54	+ if(SettingsServer::isTrackerApiRequest()) {
55	+ // if Piwik is not installed yet, the piwik.php should do nothing and not return an error
56	+ throw new NotYetInstalledException("As Matomo is not installed yet, the Tracking API cannot proceed and will exit without error.");
57	+ }
58	+
59	+ if(Common::isPhpCliMode()) {
60	+ // in CLI, do not start/redirect to installer, simply output the exception at the top
61	+ $startInstaller = false;
62	+ }
63	+
64	+ // Start the installation when config file not found
65	+ $this->checkConfigFileExists($this->settingsProvider->getPathLocal(), $startInstaller);
66	+
67	+ }
68	+
69	+ /**
70	+ * @param $path
71	+ * @param bool $startInstaller
72	+ * @throws \Exception
73	+ */
74	+ private function checkConfigFileExists($path, $startInstaller = false)
75	+ {
76	+ if (is_readable($path)) {
77	+ return;
78	+ }
79	+
80	+ $general = $this->settingsProvider->getSection('General');
81	+
82	+ if (isset($general['enable_installer'])
83	+ && !$general['enable_installer']
84	+ ) {
85	+ throw new NotYetInstalledException('Matomo is not set up yet');
86	+ }
87	+
88	+ $message = $this->getSpecificMessageWhetherFileExistsOrNot($path);
89	+
90	+ $exception = new NotYetInstalledException($message);
91	+
92	+ if ($startInstaller) {
93	+ $this->startInstallation($exception);
94	+ } else {
95	+ throw $exception;
96	+ }
97	+ }
98	+
99	+ /**
100	+ * @param $exception
101	+ */
102	+ private function startInstallation($exception)
103	+ {
104	+ /**
105	+ * Triggered when the configuration file cannot be found or read, which usually
106	+ * means Piwik is not installed yet.
107	+ *
108	+ * This event can be used to start the installation process or to display a custom error message.
109	+ *
110	+ * @param \Exception $exception The exception that was thrown by `Config::getInstance()`.
111	+ */
112	+ Piwik::postEvent('Config.NoConfigurationFile', array($exception), $pending = true);
113	+ }
114	+
115	+ /**
116	+ * @param $path
117	+ * @return string
118	+ */
119	+ private function getMessageWhenFileExistsButNotReadable($path)
120	+ {
121	+ $format = " \n<b>» %s </b>";
122	+ if(Common::isPhpCliMode()) {
123	+ $format = "\n » %s \n";
124	+ }
125	+
126	+ return sprintf($format,
127	+ $this->translator->translate('General_ExceptionConfigurationFilePleaseCheckReadableByUser',
128	+ array($path, Filechecks::getUser())));
129	+ }
130	+
131	+ /**
132	+ * @param $path
133	+ * @return string
134	+ */
135	+ private function getSpecificMessageWhetherFileExistsOrNot($path)
136	+ {
137	+ if (!file_exists($path)) {
138	+ $message = $this->translator->translate('General_ExceptionConfigurationFileNotFound', array($path));
139	+ if (Common::isPhpCliMode()) {
140	+ $message .= $this->getMessageWhenFileExistsButNotReadable($path);
141	+ }
142	+ } else {
143	+ $message = $this->translator->translate('General_ExceptionConfigurationFileExistsButNotReadable',
144	+ array($path));
145	+ $message .= $this->getMessageWhenFileExistsButNotReadable($path);
146	+ }
147	+
148	+ if (Common::isPhpCliMode()) {
149	+ $message = "\n" . $message;
150	+ }
151	+ return $message;
152	+ }
153	+ }

app/core/Application/Kernel/GlobalSettingsProvider.php ADDED Viewed

	@@ -0,0 +1,111 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ */
8	+
9	+ namespace Piwik\Application\Kernel;
10	+
11	+ use Piwik\Config;
12	+ use Piwik\Config\IniFileChain;
13	+
14	+ /**
15	+ * Provides global settings. Global settings are organized in sections where
16	+ * each section contains a list of name => value pairs. Setting values can
17	+ * be primitive values or arrays of primitive values.
18	+ *
19	+ * Uses the config.ini.php, common.ini.php and global.ini.php files to provide global settings.
20	+ *
21	+ * At the moment a singleton instance of this class is used in order to get tests to pass.
22	+ */
23	+ class GlobalSettingsProvider
24	+ {
25	+ /**
26	+ * @var IniFileChain
27	+ */
28	+ protected $iniFileChain;
29	+
30	+ /**
31	+ * @var string
32	+ */
33	+ protected $pathGlobal = null;
34	+
35	+ /**
36	+ * @var string
37	+ */
38	+ protected $pathCommon = null;
39	+
40	+ /**
41	+ * @var string
42	+ */
43	+ protected $pathLocal = null;
44	+
45	+ /**
46	+ * @param string\|null $pathGlobal Path to the global.ini.php file. Or null to use the default.
47	+ * @param string\|null $pathLocal Path to the config.ini.php file. Or null to use the default.
48	+ * @param string\|null $pathCommon Path to the common.ini.php file. Or null to use the default.
49	+ */
50	+ public function __construct($pathGlobal = null, $pathLocal = null, $pathCommon = null)
51	+ {
52	+ $this->pathGlobal = $pathGlobal ?: Config::getGlobalConfigPath();
53	+ $this->pathCommon = $pathCommon ?: Config::getCommonConfigPath();
54	+ $this->pathLocal = $pathLocal ?: Config::getLocalConfigPath();
55	+
56	+ $this->iniFileChain = new IniFileChain();
57	+ $this->reload();
58	+ }
59	+
60	+ public function reload($pathGlobal = null, $pathLocal = null, $pathCommon = null)
61	+ {
62	+ $this->pathGlobal = $pathGlobal ?: $this->pathGlobal;
63	+ $this->pathCommon = $pathCommon ?: $this->pathCommon;
64	+ $this->pathLocal = $pathLocal ?: $this->pathLocal;
65	+
66	+ $this->iniFileChain->reload(array($this->pathGlobal, $this->pathCommon), $this->pathLocal);
67	+ }
68	+
69	+ /**
70	+ * Returns a settings section.
71	+ *
72	+ * @param string $name
73	+ * @return array
74	+ */
75	+ public function &getSection($name)
76	+ {
77	+ $section =& $this->iniFileChain->get($name);
78	+ return $section;
79	+ }
80	+
81	+ /**
82	+ * Sets a settings section.
83	+ *
84	+ * @param string $name
85	+ * @param array $value
86	+ */
87	+ public function setSection($name, $value)
88	+ {
89	+ $this->iniFileChain->set($name, $value);
90	+ }
91	+
92	+ public function getIniFileChain()
93	+ {
94	+ return $this->iniFileChain;
95	+ }
96	+
97	+ public function getPathGlobal()
98	+ {
99	+ return $this->pathGlobal;
100	+ }
101	+
102	+ public function getPathLocal()
103	+ {
104	+ return $this->pathLocal;
105	+ }
106	+
107	+ public function getPathCommon()
108	+ {
109	+ return $this->pathCommon;
110	+ }
111	+ }

app/core/Application/Kernel/PluginList.php ADDED Viewed

	@@ -0,0 +1,197 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ */
8	+
9	+ namespace Piwik\Application\Kernel;
10	+
11	+ use Piwik\Plugin\MetadataLoader;
12	+
13	+ /**
14	+ * Lists the currently activated plugins. Used when setting up Piwik's environment before
15	+ * initializing the DI container.
16	+ *
17	+ * Uses the [Plugins] section in Piwik's INI config to get the activated plugins.
18	+ *
19	+ * Depends on GlobalSettingsProvider being used.
20	+ *
21	+ * TODO: parts of Plugin\Manager edit the plugin list; maybe PluginList implementations should be mutable?
22	+ */
23	+ class PluginList
24	+ {
25	+ /**
26	+ * @var GlobalSettingsProvider
27	+ */
28	+ private $settings;
29	+
30	+ /**
31	+ * Plugins bundled with core package, disabled by default
32	+ * @var array
33	+ */
34	+ private $corePluginsDisabledByDefault = array(
35	+ 'DBStats',
36	+ 'ExamplePlugin',
37	+ 'ExampleCommand',
38	+ 'ExampleSettingsPlugin',
39	+ 'ExampleUI',
40	+ 'ExampleVisualization',
41	+ 'ExamplePluginTemplate',
42	+ 'ExampleTracker',
43	+ 'ExampleLogTables',
44	+ 'ExampleReport',
45	+ 'ExampleAPI',
46	+ 'MobileAppMeasurable',
47	+ 'Provider',
48	+ 'TagManager'
49	+ );
50	+
51	+ // Themes bundled with core package, disabled by default
52	+ private $coreThemesDisabledByDefault = array(
53	+ 'ExampleTheme'
54	+ );
55	+
56	+ public function __construct(GlobalSettingsProvider $settings)
57	+ {
58	+ $this->settings = $settings;
59	+ }
60	+
61	+ /**
62	+ * Returns the list of plugins that should be loaded. Used by the container factory to
63	+ * load plugin specific DI overrides.
64	+ *
65	+ * @return string[]
66	+ */
67	+ public function getActivatedPlugins()
68	+ {
69	+ $section = $this->settings->getSection('Plugins');
70	+ $plugins = @$section['Plugins'] ?: array();
71	+
72	+ return $plugins;
73	+ }
74	+
75	+ /**
76	+ * Returns the list of plugins that are bundled with Piwik.
77	+ *
78	+ * @return string[]
79	+ */
80	+ public function getPluginsBundledWithPiwik()
81	+ {
82	+ $pathGlobal = $this->settings->getPathGlobal();
83	+
84	+ $section = $this->settings->getIniFileChain()->getFrom($pathGlobal, 'Plugins');
85	+ return $section['Plugins'];
86	+ }
87	+
88	+ /**
89	+ * Returns the plugins bundled with core package that are disabled by default.
90	+ *
91	+ * @return string[]
92	+ */
93	+ public function getCorePluginsDisabledByDefault()
94	+ {
95	+ return array_merge($this->corePluginsDisabledByDefault, $this->coreThemesDisabledByDefault);
96	+ }
97	+
98	+ /**
99	+ * Sorts an array of plugins in the order they should be loaded. We cannot use DI here as DI is not initialized
100	+ * at this stage.
101	+ *
102	+ * @params string[] $plugins
103	+ * @return \string[]
104	+ */
105	+ public function sortPlugins(array $plugins)
106	+ {
107	+ $global = $this->getPluginsBundledWithPiwik();
108	+ if (empty($global)) {
109	+ return $plugins;
110	+ }
111	+
112	+ // we need to make sure a possibly disabled plugin will be still loaded before any 3rd party plugin
113	+ $global = array_merge($global, $this->corePluginsDisabledByDefault);
114	+
115	+ $global = array_values($global);
116	+ $plugins = array_values($plugins);
117	+
118	+ $defaultPluginsLoadedFirst = array_intersect($global, $plugins);
119	+
120	+ $otherPluginsToLoadAfterDefaultPlugins = array_diff($plugins, $defaultPluginsLoadedFirst);
121	+
122	+ // sort by name to have a predictable order for those extra plugins
123	+ natcasesort($otherPluginsToLoadAfterDefaultPlugins);
124	+
125	+ $sorted = array_merge($defaultPluginsLoadedFirst, $otherPluginsToLoadAfterDefaultPlugins);
126	+
127	+ return $sorted;
128	+ }
129	+
130	+ /**
131	+ * Sorts an array of plugins in the order they should be saved in config.ini.php. This basically influences
132	+ * the order of the plugin config.php and which config will be loaded first. We want to make sure to require the
133	+ * config or a required plugin first before loading the plugin that requires it.
134	+ *
135	+ * We do not sort using this logic on each request since it is much slower than `sortPlugins()`. The order
136	+ * of plugins in config.ini.php is only important for the ContainerFactory. During a regular request it is otherwise
137	+ * fine to load the plugins in the order of `sortPlugins()` since we will make sure that required plugins will be
138	+ * loaded first in plugin manager.
139	+ *
140	+ * @param string[] $plugins
141	+ * @param array[] $pluginJsonCache For internal testing only
142	+ * @return \string[]
143	+ */
144	+ public function sortPluginsAndRespectDependencies(array $plugins, $pluginJsonCache = array())
145	+ {
146	+ $global = $this->getPluginsBundledWithPiwik();
147	+
148	+ if (empty($global)) {
149	+ return $plugins;
150	+ }
151	+
152	+ // we need to make sure a possibly disabled plugin will be still loaded before any 3rd party plugin
153	+ $global = array_merge($global, $this->corePluginsDisabledByDefault);
154	+
155	+ $global = array_values($global);
156	+ $plugins = array_values($plugins);
157	+
158	+ $defaultPluginsLoadedFirst = array_intersect($global, $plugins);
159	+
160	+ $otherPluginsToLoadAfterDefaultPlugins = array_diff($plugins, $defaultPluginsLoadedFirst);
161	+
162	+ // we still want to sort alphabetically by default
163	+ natcasesort($otherPluginsToLoadAfterDefaultPlugins);
164	+
165	+ $sorted = array();
166	+ foreach ($otherPluginsToLoadAfterDefaultPlugins as $pluginName) {
167	+ $sorted = $this->sortRequiredPlugin($pluginName, $pluginJsonCache, $otherPluginsToLoadAfterDefaultPlugins, $sorted);
168	+ }
169	+
170	+ $sorted = array_merge($defaultPluginsLoadedFirst, $sorted);
171	+
172	+ return $sorted;
173	+ }
174	+
175	+ private function sortRequiredPlugin($pluginName, &$pluginJsonCache, $toBeSorted, $sorted)
176	+ {
177	+ if (!isset($pluginJsonCache[$pluginName])) {
178	+ $loader = new MetadataLoader($pluginName);
179	+ $pluginJsonCache[$pluginName] = $loader->loadPluginInfoJson();
180	+ }
181	+
182	+ if (!empty($pluginJsonCache[$pluginName]['require'])) {
183	+ $dependencies = $pluginJsonCache[$pluginName]['require'];
184	+ foreach ($dependencies as $possiblePluginName => $key) {
185	+ if (in_array($possiblePluginName, $toBeSorted, true) && !in_array($possiblePluginName, $sorted, true)) {
186	+ $sorted = $this->sortRequiredPlugin($possiblePluginName, $pluginJsonCache, $toBeSorted, $sorted);
187	+ }
188	+ }
189	+ }
190	+
191	+ if (!in_array($pluginName, $sorted, true)) {
192	+ $sorted[] = $pluginName;
193	+ }
194	+
195	+ return $sorted;
196	+ }
197	+ }

app/core/Archive.php ADDED Viewed

	@@ -0,0 +1,906 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik;
10	+
11	+ use Piwik\Archive\ArchiveQuery;
12	+ use Piwik\Archive\ArchiveQueryFactory;
13	+ use Piwik\Archive\Parameters;
14	+ use Piwik\ArchiveProcessor\Rules;
15	+ use Piwik\Archive\ArchiveInvalidator;
16	+ use Piwik\Container\StaticContainer;
17	+ use Piwik\DataAccess\ArchiveSelector;
18	+
19	+ /**
20	+ * The Archive class is used to query cached analytics statistics
21	+ * (termed "archive data").
22	+ *
23	+ * You can use Archive instances to get data that was archived for one or more sites,
24	+ * for one or more periods and one optional segment.
25	+ *
26	+ * If archive data is not found, this class will initiate the archiving process. [1](#footnote-1)
27	+ *
28	+ * Archive instances must be created using the {@link build()} factory method;
29	+ * they cannot be constructed.
30	+ *
31	+ * You can search for metrics (such as `nb_visits`) using the {@link getNumeric()} and
32	+ * {@link getDataTableFromNumeric()} methods. You can search for
33	+ * reports using the {@link getBlob()}, {@link getDataTable()} and {@link getDataTableExpanded()} methods.
34	+ *
35	+ * If you're creating an API that returns report data, you may want to use the
36	+ * {@link createDataTableFromArchive()} helper function.
37	+ *
38	+ * ### Learn more
39	+ *
40	+ * Learn more about _archiving_ [here](/guides/all-about-analytics-data).
41	+ *
42	+ * ### Limitations
43	+ *
44	+ * - You cannot get data for multiple range periods in a single query.
45	+ * - You cannot get data for periods of different types in a single query.
46	+ *
47	+ * ### Examples
48	+ *
49	+ * _Querying metrics for an API method_
50	+ *
51	+ * // one site and one period
52	+ * $archive = Archive::build($idSite = 1, $period = 'week', $date = '2013-03-08');
53	+ * return $archive->getDataTableFromNumeric(array('nb_visits', 'nb_actions'));
54	+ *
55	+ * // all sites and multiple dates
56	+ * $archive = Archive::build($idSite = 'all', $period = 'month', $date = '2013-01-02,2013-03-08');
57	+ * return $archive->getDataTableFromNumeric(array('nb_visits', 'nb_actions'));
58	+ *
59	+ * _Querying and using metrics immediately_
60	+ *
61	+ * // one site and one period
62	+ * $archive = Archive::build($idSite = 1, $period = 'week', $date = '2013-03-08');
63	+ * $data = $archive->getNumeric(array('nb_visits', 'nb_actions'));
64	+ *
65	+ * $visits = $data['nb_visits'];
66	+ * $actions = $data['nb_actions'];
67	+ *
68	+ * // ... do something w/ metric data ...
69	+ *
70	+ * // multiple sites and multiple dates
71	+ * $archive = Archive::build($idSite = '1,2,3', $period = 'month', $date = '2013-01-02,2013-03-08');
72	+ * $data = $archive->getNumeric('nb_visits');
73	+ *
74	+ * $janSite1Visits = $data['1']['2013-01-01,2013-01-31']['nb_visits'];
75	+ * $febSite1Visits = $data['1']['2013-02-01,2013-02-28']['nb_visits'];
76	+ * // ... etc.
77	+ *
78	+ * _Querying for reports_
79	+ *
80	+ * $archive = Archive::build($idSite = 1, $period = 'week', $date = '2013-03-08');
81	+ * $dataTable = $archive->getDataTable('MyPlugin_MyReport');
82	+ * // ... manipulate $dataTable ...
83	+ * return $dataTable;
84	+ *
85	+ * _Querying a report for an API method_
86	+ *
87	+ * public function getMyReport($idSite, $period, $date, $segment = false, $expanded = false)
88	+ * {
89	+ * $dataTable = Archive::createDataTableFromArchive('MyPlugin_MyReport', $idSite, $period, $date, $segment, $expanded);
90	+ * return $dataTable;
91	+ * }
92	+ *
93	+ * _Querying data for multiple range periods_
94	+ *
95	+ * // get data for first range
96	+ * $archive = Archive::build($idSite = 1, $period = 'range', $date = '2013-03-08,2013-03-12');
97	+ * $dataTable = $archive->getDataTableFromNumeric(array('nb_visits', 'nb_actions'));
98	+ *
99	+ * // get data for second range
100	+ * $archive = Archive::build($idSite = 1, $period = 'range', $date = '2013-03-15,2013-03-20');
101	+ * $dataTable = $archive->getDataTableFromNumeric(array('nb_visits', 'nb_actions'));
102	+ *
103	+ * <a name="footnote-1"></a>
104	+ * [1]: The archiving process will not be launched if browser archiving is disabled
105	+ * and the current request came from a browser.
106	+ *
107	+ *
108	+ * @api
109	+ */
110	+ class Archive implements ArchiveQuery
111	+ {
112	+ const REQUEST_ALL_WEBSITES_FLAG = 'all';
113	+ const ARCHIVE_ALL_PLUGINS_FLAG = 'all';
114	+ const ID_SUBTABLE_LOAD_ALL_SUBTABLES = 'all';
115	+
116	+ /**
117	+ * List of archive IDs for the site, periods and segment we are querying with.
118	+ * Archive IDs are indexed by done flag and period, ie:
119	+ *
120	+ * array(
121	+ * 'done.Referrers' => array(
122	+ * '2010-01-01' => 1,
123	+ * '2010-01-02' => 2,
124	+ * ),
125	+ * 'done.VisitsSummary' => array(
126	+ * '2010-01-01' => 3,
127	+ * '2010-01-02' => 4,
128	+ * ),
129	+ * )
130	+ *
131	+ * or,
132	+ *
133	+ * array(
134	+ * 'done.all' => array(
135	+ * '2010-01-01' => 1,
136	+ * '2010-01-02' => 2
137	+ * )
138	+ * )
139	+ *
140	+ * @var array
141	+ */
142	+ private $idarchives = array();
143	+
144	+ /**
145	+ * If set to true, the result of all get functions (ie, getNumeric, getBlob, etc.)
146	+ * will be indexed by the site ID, even if we're only querying data for one site.
147	+ *
148	+ * @var bool
149	+ */
150	+ private $forceIndexedBySite;
151	+
152	+ /**
153	+ * If set to true, the result of all get functions (ie, getNumeric, getBlob, etc.)
154	+ * will be indexed by the period, even if we're only querying data for one period.
155	+ *
156	+ * @var bool
157	+ */
158	+ private $forceIndexedByDate;
159	+
160	+ /**
161	+ * @var Parameters
162	+ */
163	+ private $params;
164	+
165	+ /**
166	+ * @var \Piwik\Cache\Cache
167	+ */
168	+ private static $cache;
169	+
170	+ /**
171	+ * @var ArchiveInvalidator
172	+ */
173	+ private $invalidator;
174	+
175	+ /**
176	+ * @param Parameters $params
177	+ * @param bool $forceIndexedBySite Whether to force index the result of a query by site ID.
178	+ * @param bool $forceIndexedByDate Whether to force index the result of a query by period.
179	+ */
180	+ public function __construct(Parameters $params, $forceIndexedBySite = false,
181	+ $forceIndexedByDate = false)
182	+ {
183	+ $this->params = $params;
184	+ $this->forceIndexedBySite = $forceIndexedBySite;
185	+ $this->forceIndexedByDate = $forceIndexedByDate;
186	+
187	+ $this->invalidator = StaticContainer::get('Piwik\Archive\ArchiveInvalidator');
188	+ }
189	+
190	+ /**
191	+ * Returns a new Archive instance that will query archive data for the given set of
192	+ * sites and periods, using an optional Segment.
193	+ *
194	+ * This method uses data that is found in query parameters, so the parameters to this
195	+ * function can be string values.
196	+ *
197	+ * If you want to create an Archive instance with an array of Period instances, use
198	+ * {@link Archive::factory()}.
199	+ *
200	+ * @param string\|int\|array $idSites A single ID (eg, `'1'`), multiple IDs (eg, `'1,2,3'` or `array(1, 2, 3)`),
201	+ * or `'all'`.
202	+ * @param string $period 'day', `'week'`, `'month'`, `'year'` or `'range'`
203	+ * @param Date\|string $strDate 'YYYY-MM-DD', magic keywords (ie, 'today'; {@link Date::factory()}
204	+ * or date range (ie, 'YYYY-MM-DD,YYYY-MM-DD').
205	+ * @param bool\|false\|string $segment Segment definition or false if no segment should be used. {@link Piwik\Segment}
206	+ * @param bool\|false\|string $_restrictSitesToLogin Used only when running as a scheduled task.
207	+ * @return ArchiveQuery
208	+ */
209	+ public static function build($idSites, $period, $strDate, $segment = false, $_restrictSitesToLogin = false)
210	+ {
211	+ return StaticContainer::get(ArchiveQueryFactory::class)->build($idSites, $period, $strDate, $segment,
212	+ $_restrictSitesToLogin);
213	+ }
214	+
215	+ /**
216	+ * Returns a new Archive instance that will query archive data for the given set of
217	+ * sites and periods, using an optional segment.
218	+ *
219	+ * This method uses an array of Period instances and a Segment instance, instead of strings
220	+ * like {@link build()}.
221	+ *
222	+ * If you want to create an Archive instance using data found in query parameters,
223	+ * use {@link build()}.
224	+ *
225	+ * @param Segment $segment The segment to use. For no segment, use `new Segment('', $idSites)`.
226	+ * @param array $periods An array of Period instances.
227	+ * @param array $idSites An array of site IDs (eg, `array(1, 2, 3)`).
228	+ * @param bool $idSiteIsAll Whether `'all'` sites are being queried or not. If true, then
229	+ * the result of querying functions will be indexed by site, regardless
230	+ * of whether `count($idSites) == 1`.
231	+ * @param bool $isMultipleDate Whether multiple dates are being queried or not. If true, then
232	+ * the result of querying functions will be indexed by period,
233	+ * regardless of whether `count($periods) == 1`.
234	+ *
235	+ * @return ArchiveQuery
236	+ */
237	+ public static function factory(Segment $segment, array $periods, array $idSites, $idSiteIsAll = false,
238	+ $isMultipleDate = false)
239	+ {
240	+ return StaticContainer::get(ArchiveQueryFactory::class)->factory($segment, $periods, $idSites, $idSiteIsAll,
241	+ $isMultipleDate);
242	+ }
243	+
244	+ /**
245	+ * Queries and returns metric data in an array.
246	+ *
247	+ * If multiple sites were requested in {@link build()} or {@link factory()} the result will
248	+ * be indexed by site ID.
249	+ *
250	+ * If multiple periods were requested in {@link build()} or {@link factory()} the result will
251	+ * be indexed by period.
252	+ *
253	+ * The site ID index is always first, so if multiple sites & periods were requested, the result
254	+ * will be indexed by site ID first, then period.
255	+ *
256	+ * @param string\|array $names One or more archive names, eg, `'nb_visits'`, `'Referrers_distinctKeywords'`,
257	+ * etc.
258	+ * @return false\|integer\|array `false` if there is no data to return, a single numeric value if we're not querying
259	+ * for multiple sites/periods, or an array if multiple sites, periods or names are
260	+ * queried for.
261	+ */
262	+ public function getNumeric($names)
263	+ {
264	+ $data = $this->get($names, 'numeric');
265	+
266	+ $resultIndices = $this->getResultIndices();
267	+ $result = $data->getIndexedArray($resultIndices);
268	+
269	+ // if only one metric is returned, just return it as a numeric value
270	+ if (empty($resultIndices)
271	+ && count($result) <= 1
272	+ && (!is_array($names) \|\| count($names) == 1)
273	+ ) {
274	+ $result = (float)reset($result); // convert to float in case $result is empty
275	+ }
276	+
277	+ return $result;
278	+ }
279	+
280	+ /**
281	+ * Queries and returns metric data in a DataTable instance.
282	+ *
283	+ * If multiple sites were requested in {@link build()} or {@link factory()} the result will
284	+ * be a DataTable\Map that is indexed by site ID.
285	+ *
286	+ * If multiple periods were requested in {@link build()} or {@link factory()} the result will
287	+ * be a {@link DataTable\Map} that is indexed by period.
288	+ *
289	+ * The site ID index is always first, so if multiple sites & periods were requested, the result
290	+ * will be a {@link DataTable\Map} indexed by site ID which contains {@link DataTable\Map} instances that are
291	+ * indexed by period.
292	+ *
293	+ * _Note: Every DataTable instance returned will have at most one row in it. The contents of each
294	+ * row will be the requested metrics for the appropriate site and period._
295	+ *
296	+ * @param string\|array $names One or more archive names, eg, 'nb_visits', 'Referrers_distinctKeywords',
297	+ * etc.
298	+ * @return DataTable\|DataTable\Map A DataTable if multiple sites and periods were not requested.
299	+ * An appropriately indexed DataTable\Map if otherwise.
300	+ */
301	+ public function getDataTableFromNumeric($names)
302	+ {
303	+ $data = $this->get($names, 'numeric');
304	+ return $data->getDataTable($this->getResultIndices());
305	+ }
306	+
307	+ /**
308	+ * Similar to {@link getDataTableFromNumeric()} but merges all children on the created DataTable.
309	+ *
310	+ * This is the same as doing `$this->getDataTableFromNumeric()->mergeChildren()` but this way it is much faster.
311	+ *
312	+ * @return DataTable\|DataTable\Map
313	+ *
314	+ * @internal Currently only used by MultiSites.getAll plugin. Feel free to remove internal tag if needed somewhere
315	+ * else. If no longer needed by MultiSites.getAll please remove this method. If you need this to work in
316	+ * a bit different way feel free to refactor as always.
317	+ */
318	+ public function getDataTableFromNumericAndMergeChildren($names)
319	+ {
320	+ $data = $this->get($names, 'numeric');
321	+ $resultIndexes = $this->getResultIndices();
322	+ return $data->getMergedDataTable($resultIndexes);
323	+ }
324	+
325	+ /**
326	+ * Queries and returns one or more reports as DataTable instances.
327	+ *
328	+ * This method will query blob data that is a serialized array of of {@link DataTable\Row}'s and
329	+ * unserialize it.
330	+ *
331	+ * If multiple sites were requested in {@link build()} or {@link factory()} the result will
332	+ * be a {@link DataTable\Map} that is indexed by site ID.
333	+ *
334	+ * If multiple periods were requested in {@link build()} or {@link factory()} the result will
335	+ * be a DataTable\Map that is indexed by period.
336	+ *
337	+ * The site ID index is always first, so if multiple sites & periods were requested, the result
338	+ * will be a {@link DataTable\Map} indexed by site ID which contains {@link DataTable\Map} instances that are
339	+ * indexed by period.
340	+ *
341	+ * @param string $name The name of the record to get. This method can only query one record at a time.
342	+ * @param int\|string\|null $idSubtable The ID of the subtable to get (if any).
343	+ * @return DataTable\|DataTable\Map A DataTable if multiple sites and periods were not requested.
344	+ * An appropriately indexed {@link DataTable\Map} if otherwise.
345	+ */
346	+ public function getDataTable($name, $idSubtable = null)
347	+ {
348	+ $data = $this->get($name, 'blob', $idSubtable);
349	+ return $data->getDataTable($this->getResultIndices());
350	+ }
351	+
352	+ /**
353	+ * Queries and returns one report with all of its subtables loaded.
354	+ *
355	+ * If multiple sites were requested in {@link build()} or {@link factory()} the result will
356	+ * be a DataTable\Map that is indexed by site ID.
357	+ *
358	+ * If multiple periods were requested in {@link build()} or {@link factory()} the result will
359	+ * be a DataTable\Map that is indexed by period.
360	+ *
361	+ * The site ID index is always first, so if multiple sites & periods were requested, the result
362	+ * will be a {@link DataTable\Map indexed} by site ID which contains {@link DataTable\Map} instances that are
363	+ * indexed by period.
364	+ *
365	+ * @param string $name The name of the record to get.
366	+ * @param int\|string\|null $idSubtable The ID of the subtable to get (if any). The subtable will be expanded.
367	+ * @param int\|null $depth The maximum number of subtable levels to load. If null, all levels are loaded.
368	+ * For example, if `1` is supplied, then the DataTable returned will have its subtables
369	+ * loaded. Those subtables, however, will NOT have their subtables loaded.
370	+ * @param bool $addMetadataSubtableId Whether to add the database subtable ID as metadata to each datatable,
371	+ * or not.
372	+ * @return DataTable\|DataTable\Map
373	+ */
374	+ public function getDataTableExpanded($name, $idSubtable = null, $depth = null, $addMetadataSubtableId = true)
375	+ {
376	+ $data = $this->get($name, 'blob', self::ID_SUBTABLE_LOAD_ALL_SUBTABLES);
377	+ return $data->getExpandedDataTable($this->getResultIndices(), $idSubtable, $depth, $addMetadataSubtableId);
378	+ }
379	+
380	+ /**
381	+ * Returns the list of plugins that archive the given reports.
382	+ *
383	+ * @param array $archiveNames
384	+ * @return array
385	+ */
386	+ private function getRequestedPlugins($archiveNames)
387	+ {
388	+ $result = array();
389	+
390	+ foreach ($archiveNames as $name) {
391	+ $result[] = self::getPluginForReport($name);
392	+ }
393	+
394	+ return array_unique($result);
395	+ }
396	+
397	+ /**
398	+ * Returns an object describing the set of sites, the set of periods and the segment
399	+ * this Archive will query data for.
400	+ *
401	+ * @return Parameters
402	+ */
403	+ public function getParams()
404	+ {
405	+ return $this->params;
406	+ }
407	+
408	+ /**
409	+ * Helper function that creates an Archive instance and queries for report data using
410	+ * query parameter data. API methods can use this method to reduce code redundancy.
411	+ *
412	+ * @param string $recordName The name of the report to return.
413	+ * @param int\|string\|array $idSite @see {@link build()}
414	+ * @param string $period @see {@link build()}
415	+ * @param string $date @see {@link build()}
416	+ * @param string $segment @see {@link build()}
417	+ * @param bool $expanded If true, loads all subtables. See {@link getDataTableExpanded()}
418	+ * @param bool $flat If true, loads all subtables and disabled all recursive filters.
419	+ * @param int\|null $idSubtable See {@link getDataTableExpanded()}
420	+ * @param int\|null $depth See {@link getDataTableExpanded()}
421	+ * @return DataTable\|DataTable\Map
422	+ */
423	+ public static function createDataTableFromArchive($recordName, $idSite, $period, $date, $segment, $expanded = false, $flat = false, $idSubtable = null, $depth = null)
424	+ {
425	+ Piwik::checkUserHasViewAccess($idSite);
426	+
427	+ if ($flat && !$idSubtable) {
428	+ $expanded = true;
429	+ }
430	+
431	+ $archive = Archive::build($idSite, $period, $date, $segment, $_restrictSitesToLogin = false);
432	+ if ($idSubtable === false) {
433	+ $idSubtable = null;
434	+ }
435	+
436	+ if ($expanded) {
437	+ $dataTable = $archive->getDataTableExpanded($recordName, $idSubtable, $depth);
438	+ } else {
439	+ $dataTable = $archive->getDataTable($recordName, $idSubtable);
440	+ }
441	+
442	+ $dataTable->queueFilter('ReplaceSummaryRowLabel');
443	+ $dataTable->queueFilter('ReplaceColumnNames');
444	+
445	+ if ($expanded) {
446	+ $dataTable->queueFilterSubtables('ReplaceColumnNames');
447	+ }
448	+
449	+ if ($flat) {
450	+ $dataTable->disableRecursiveFilters();
451	+ }
452	+
453	+ return $dataTable;
454	+ }
455	+
456	+ private function getSiteIdsThatAreRequestedInThisArchiveButWereNotInvalidatedYet()
457	+ {
458	+ if (is_null(self::$cache)) {
459	+ self::$cache = Cache::getTransientCache();
460	+ }
461	+
462	+ $id = 'Archive.SiteIdsOfRememberedReportsInvalidated';
463	+
464	+ if (!self::$cache->contains($id)) {
465	+ self::$cache->save($id, array());
466	+ }
467	+
468	+ $siteIdsAlreadyHandled = self::$cache->fetch($id);
469	+ $siteIdsRequested = $this->params->getIdSites();
470	+
471	+ foreach ($siteIdsRequested as $index => $siteIdRequested) {
472	+ $siteIdRequested = (int) $siteIdRequested;
473	+
474	+ if (in_array($siteIdRequested, $siteIdsAlreadyHandled)) {
475	+ unset($siteIdsRequested[$index]); // was already handled previously, do not do it again
476	+ } else {
477	+ $siteIdsAlreadyHandled[] = $siteIdRequested; // we will handle this id this time
478	+ }
479	+ }
480	+
481	+ self::$cache->save($id, $siteIdsAlreadyHandled);
482	+
483	+ return $siteIdsRequested;
484	+ }
485	+
486	+ private function invalidatedReportsIfNeeded()
487	+ {
488	+ $siteIdsRequested = $this->getSiteIdsThatAreRequestedInThisArchiveButWereNotInvalidatedYet();
489	+
490	+ if (empty($siteIdsRequested)) {
491	+ return; // all requested site ids were already handled
492	+ }
493	+
494	+ $sitesPerDays = $this->invalidator->getRememberedArchivedReportsThatShouldBeInvalidated();
495	+
496	+ foreach ($sitesPerDays as $date => $siteIds) {
497	+ if (empty($siteIds)) {
498	+ continue;
499	+ }
500	+
501	+ $siteIdsToActuallyInvalidate = array_intersect($siteIds, $siteIdsRequested);
502	+
503	+ if (empty($siteIdsToActuallyInvalidate)) {
504	+ continue; // all site ids that should be handled are already handled
505	+ }
506	+
507	+ try {
508	+ $this->invalidator->markArchivesAsInvalidated($siteIdsToActuallyInvalidate, array(Date::factory($date)), false);
509	+ } catch (\Exception $e) {
510	+ Site::clearCache();
511	+ throw $e;
512	+ }
513	+ }
514	+
515	+ Site::clearCache();
516	+ }
517	+
518	+ /**
519	+ * Queries archive tables for data and returns the result.
520	+ * @param array\|string $archiveNames
521	+ * @param $archiveDataType
522	+ * @param null\|int $idSubtable
523	+ * @return Archive\DataCollection
524	+ */
525	+ protected function get($archiveNames, $archiveDataType, $idSubtable = null)
526	+ {
527	+ if (!is_array($archiveNames)) {
528	+ $archiveNames = array($archiveNames);
529	+ }
530	+
531	+ // apply idSubtable
532	+ if ($idSubtable !== null
533	+ && $idSubtable != self::ID_SUBTABLE_LOAD_ALL_SUBTABLES
534	+ ) {
535	+ // this is also done in ArchiveSelector. It should be actually only done in ArchiveSelector but DataCollection
536	+ // does require to have the subtableId appended. Needs to be changed in refactoring to have it only in one
537	+ // place.
538	+ $dataNames = array();
539	+ foreach ($archiveNames as $name) {
540	+ $dataNames[] = ArchiveSelector::appendIdsubtable($name, $idSubtable);
541	+ }
542	+ } else {
543	+ $dataNames = $archiveNames;
544	+ }
545	+
546	+ $result = new Archive\DataCollection(
547	+ $dataNames, $archiveDataType, $this->params->getIdSites(), $this->params->getPeriods(), $this->params->getSegment(), $defaultRow = null);
548	+
549	+ $archiveIds = $this->getArchiveIds($archiveNames);
550	+ if (empty($archiveIds)) {
551	+ /**
552	+ * Triggered when no archive data is found in an API request.
553	+ * @ignore
554	+ */
555	+ Piwik::postEvent('Archive.noArchivedData');
556	+ return $result;
557	+ }
558	+
559	+ $archiveData = ArchiveSelector::getArchiveData($archiveIds, $archiveNames, $archiveDataType, $idSubtable);
560	+
561	+ $isNumeric = $archiveDataType == 'numeric';
562	+
563	+ foreach ($archiveData as $row) {
564	+ // values are grouped by idsite (site ID), date1-date2 (date range), then name (field name)
565	+ $periodStr = $row['date1'] . ',' . $row['date2'];
566	+
567	+ if ($isNumeric) {
568	+ $row['value'] = $this->formatNumericValue($row['value']);
569	+ } else {
570	+ $result->addMetadata($row['idsite'], $periodStr, DataTable::ARCHIVED_DATE_METADATA_NAME, $row['ts_archived']);
571	+ }
572	+
573	+ $result->set($row['idsite'], $periodStr, $row['name'], $row['value']);
574	+ }
575	+
576	+ return $result;
577	+ }
578	+
579	+ /**
580	+ * Returns archive IDs for the sites, periods and archive names that are being
581	+ * queried. This function will use the idarchive cache if it has the right data,
582	+ * query archive tables for IDs w/o launching archiving, or launch archiving and
583	+ * get the idarchive from ArchiveProcessor instances.
584	+ *
585	+ * @param string $archiveNames
586	+ * @return array
587	+ */
588	+ private function getArchiveIds($archiveNames)
589	+ {
590	+ $plugins = $this->getRequestedPlugins($archiveNames);
591	+
592	+ // figure out which archives haven't been processed (if an archive has been processed,
593	+ // then we have the archive IDs in $this->idarchives)
594	+ $doneFlags = array();
595	+ $archiveGroups = array();
596	+ foreach ($plugins as $plugin) {
597	+ $doneFlag = $this->getDoneStringForPlugin($plugin, $this->params->getIdSites());
598	+
599	+ $doneFlags[$doneFlag] = true;
600	+ if (!isset($this->idarchives[$doneFlag])) {
601	+ $archiveGroup = $this->getArchiveGroupOfPlugin($plugin);
602	+
603	+ if ($archiveGroup == self::ARCHIVE_ALL_PLUGINS_FLAG) {
604	+ $archiveGroup = reset($plugins);
605	+ }
606	+ $archiveGroups[] = $archiveGroup;
607	+ }
608	+
609	+ $globalDoneFlag = Rules::getDoneFlagArchiveContainsAllPlugins($this->params->getSegment());
610	+ if ($globalDoneFlag !== $doneFlag) {
611	+ $doneFlags[$globalDoneFlag] = true;
612	+ }
613	+ }
614	+
615	+ $archiveGroups = array_unique($archiveGroups);
616	+
617	+ // cache id archives for plugins we haven't processed yet
618	+ if (!empty($archiveGroups)) {
619	+ if (!Rules::isArchivingDisabledFor($this->params->getIdSites(), $this->params->getSegment(), $this->getPeriodLabel())) {
620	+ $this->cacheArchiveIdsAfterLaunching($archiveGroups, $plugins);
621	+ } else {
622	+ $this->cacheArchiveIdsWithoutLaunching($plugins);
623	+ }
624	+ }
625	+
626	+ $idArchivesByMonth = $this->getIdArchivesByMonth($doneFlags);
627	+
628	+ return $idArchivesByMonth;
629	+ }
630	+
631	+ /**
632	+ * Gets the IDs of the archives we're querying for and stores them in $this->archives.
633	+ * This function will launch the archiving process for each period/site/plugin if
634	+ * metrics/reports have not been calculated/archived already.
635	+ *
636	+ * @param array $archiveGroups @see getArchiveGroupOfReport
637	+ * @param array $plugins List of plugin names to archive.
638	+ */
639	+ private function cacheArchiveIdsAfterLaunching($archiveGroups, $plugins)
640	+ {
641	+ $this->invalidatedReportsIfNeeded();
642	+
643	+ $today = Date::today();
644	+
645	+ foreach ($this->params->getPeriods() as $period) {
646	+ $twoDaysBeforePeriod = $period->getDateStart()->subDay(2);
647	+ $twoDaysAfterPeriod = $period->getDateEnd()->addDay(2);
648	+
649	+ foreach ($this->params->getIdSites() as $idSite) {
650	+ $site = new Site($idSite);
651	+
652	+ if ($period->getLabel() === 'day'
653	+ && !$this->params->getSegment()->isEmpty()
654	+ && Common::getRequestVar('skipArchiveSegmentToday', 0, 'int')
655	+ && $period->getDateStart()->toString() == Date::factory('now', $site->getTimezone())->toString()
656	+ ) {
657	+
658	+ Log::debug("Skipping archive %s for %s as segment today is disabled", $period->getLabel(), $period->getPrettyString());
659	+ continue;
660	+ }
661	+
662	+ // if the END of the period is BEFORE the website creation date
663	+ // we already know there are no stats for this period
664	+ // we add one day to make sure we don't miss the day of the website creation
665	+ if ($twoDaysAfterPeriod->isEarlier($site->getCreationDate())) {
666	+ Log::debug("Archive site %s, %s (%s) skipped, archive is before the website was created.",
667	+ $idSite, $period->getLabel(), $period->getPrettyString());
668	+ continue;
669	+ }
670	+
671	+ // if the starting date is in the future we know there is no visiidsite = ?t
672	+ if ($twoDaysBeforePeriod->isLater($today)) {
673	+ Log::debug("Archive site %s, %s (%s) skipped, archive is after today.",
674	+ $idSite, $period->getLabel(), $period->getPrettyString());
675	+ continue;
676	+ }
677	+
678	+ $this->prepareArchive($archiveGroups, $site, $period);
679	+ }
680	+ }
681	+ }
682	+
683	+ /**
684	+ * Gets the IDs of the archives we're querying for and stores them in $this->archives.
685	+ * This function will not launch the archiving process (and is thus much, much faster
686	+ * than cacheArchiveIdsAfterLaunching).
687	+ *
688	+ * @param array $plugins List of plugin names from which data is being requested.
689	+ */
690	+ private function cacheArchiveIdsWithoutLaunching($plugins)
691	+ {
692	+ $idarchivesByReport = ArchiveSelector::getArchiveIds(
693	+ $this->params->getIdSites(), $this->params->getPeriods(), $this->params->getSegment(), $plugins);
694	+
695	+ // initialize archive ID cache for each report
696	+ foreach ($plugins as $plugin) {
697	+ $doneFlag = $this->getDoneStringForPlugin($plugin, $this->params->getIdSites());
698	+ $this->initializeArchiveIdCache($doneFlag);
699	+ $globalDoneFlag = Rules::getDoneFlagArchiveContainsAllPlugins($this->params->getSegment());
700	+ $this->initializeArchiveIdCache($globalDoneFlag);
701	+ }
702	+
703	+ foreach ($idarchivesByReport as $doneFlag => $idarchivesByDate) {
704	+ foreach ($idarchivesByDate as $dateRange => $idarchives) {
705	+ foreach ($idarchives as $idarchive) {
706	+ $this->idarchives[$doneFlag][$dateRange][] = $idarchive;
707	+ }
708	+ }
709	+ }
710	+ }
711	+
712	+ /**
713	+ * Returns the done string flag for a plugin using this instance's segment & periods.
714	+ * @param string $plugin
715	+ * @return string
716	+ */
717	+ private function getDoneStringForPlugin($plugin, $idSites)
718	+ {
719	+ return Rules::getDoneStringFlagFor(
720	+ $idSites,
721	+ $this->params->getSegment(),
722	+ $this->getPeriodLabel(),
723	+ $plugin
724	+ );
725	+ }
726	+
727	+ private function getPeriodLabel()
728	+ {
729	+ $periods = $this->params->getPeriods();
730	+ return reset($periods)->getLabel();
731	+ }
732	+
733	+ /**
734	+ * Returns an array describing what metadata to use when indexing a query result.
735	+ * For use with DataCollection.
736	+ *
737	+ * @return array
738	+ */
739	+ private function getResultIndices()
740	+ {
741	+ $indices = array();
742	+
743	+ if (count($this->params->getIdSites()) > 1
744	+ \|\| $this->forceIndexedBySite
745	+ ) {
746	+ $indices['site'] = 'idSite';
747	+ }
748	+
749	+ if (count($this->params->getPeriods()) > 1
750	+ \|\| $this->forceIndexedByDate
751	+ ) {
752	+ $indices['period'] = 'date';
753	+ }
754	+
755	+ return $indices;
756	+ }
757	+
758	+ private function formatNumericValue($value)
759	+ {
760	+ // If there is no dot, we return as is
761	+ // Note: this could be an integer bigger than 32 bits
762	+ if (strpos($value, '.') === false) {
763	+ if ($value === false) {
764	+ return 0;
765	+ }
766	+ return (float)$value;
767	+ }
768	+
769	+ // Round up the value with 2 decimals
770	+ // we cast the result as float because returns false when no visitors
771	+ return round((float)$value, 2);
772	+ }
773	+
774	+ /**
775	+ * Initializes the archive ID cache ($this->idarchives) for a particular 'done' flag.
776	+ *
777	+ * It is necessary that each archive ID caching function call this method for each
778	+ * unique 'done' flag it encounters, since the getArchiveIds function determines
779	+ * whether archiving should be launched based on whether $this->idarchives has a
780	+ * an entry for a specific 'done' flag.
781	+ *
782	+ * If this function is not called, then periods with no visits will not add
783	+ * entries to the cache. If the archive is used again, SQL will be executed to
784	+ * try and find the archive IDs even though we know there are none.
785	+ *
786	+ * @param string $doneFlag
787	+ */
788	+ private function initializeArchiveIdCache($doneFlag)
789	+ {
790	+ if (!isset($this->idarchives[$doneFlag])) {
791	+ $this->idarchives[$doneFlag] = array();
792	+ }
793	+ }
794	+
795	+ /**
796	+ * Returns the archiving group identifier given a plugin.
797	+ *
798	+ * More than one plugin can be called at once when archiving. In such a case
799	+ * we don't want to launch archiving three times for three plugins if doing
800	+ * it once is enough, so getArchiveIds makes sure to get the archive group of
801	+ * all reports.
802	+ *
803	+ * If the period isn't a range, then all plugins' archiving code is executed.
804	+ * If the period is a range, then archiving code is executed individually for
805	+ * each plugin.
806	+ */
807	+ private function getArchiveGroupOfPlugin($plugin)
808	+ {
809	+ $periods = $this->params->getPeriods();
810	+ $periodLabel = reset($periods)->getLabel();
811	+
812	+ if (Rules::shouldProcessReportsAllPlugins($this->params->getIdSites(), $this->params->getSegment(), $periodLabel)) {
813	+ return self::ARCHIVE_ALL_PLUGINS_FLAG;
814	+ }
815	+
816	+ return $plugin;
817	+ }
818	+
819	+ /**
820	+ * Returns the name of the plugin that archives a given report.
821	+ *
822	+ * @param string $report Archive data name, eg, `'nb_visits'`, `'DevicesDetection_...'`, etc.
823	+ * @return string Plugin name.
824	+ * @throws \Exception If a plugin cannot be found or if the plugin for the report isn't
825	+ * activated.
826	+ */
827	+ public static function getPluginForReport($report)
828	+ {
829	+ // Core metrics are always processed in Core, for the requested date/period/segment
830	+ if (in_array($report, Metrics::getVisitsMetricNames())) {
831	+ $report = 'VisitsSummary_CoreMetrics';
832	+ } // Goal_* metrics are processed by the Goals plugin (HACK)
833	+ elseif (strpos($report, 'Goal_') === 0) {
834	+ $report = 'Goals_Metrics';
835	+ } elseif (
836	+ strrpos($report, '_returning') === strlen($report) - strlen('_returning') \|\|
837	+ strrpos($report, '_new') === strlen($report) - strlen('_new')
838	+ ) { // HACK
839	+ $report = 'VisitFrequency_Metrics';
840	+ }
841	+
842	+ $plugin = substr($report, 0, strpos($report, '_'));
843	+ if (empty($plugin)
844	+ \|\| !\Piwik\Plugin\Manager::getInstance()->isPluginActivated($plugin)
845	+ ) {
846	+ throw new \Exception("Error: The report '$report' was requested but it is not available at this stage."
847	+ . " (Plugin '$plugin' is not activated.)");
848	+ }
849	+ return $plugin;
850	+ }
851	+
852	+ /**
853	+ * @param $archiveGroups
854	+ * @param $site
855	+ * @param $period
856	+ */
857	+ private function prepareArchive(array $archiveGroups, Site $site, Period $period)
858	+ {
859	+ $parameters = new ArchiveProcessor\Parameters($site, $period, $this->params->getSegment());
860	+ $archiveLoader = new ArchiveProcessor\Loader($parameters);
861	+
862	+ $periodString = $period->getRangeString();
863	+
864	+ $idSites = array($site->getId());
865	+
866	+ // process for each plugin as well
867	+ foreach ($archiveGroups as $plugin) {
868	+ $doneFlag = $this->getDoneStringForPlugin($plugin, $idSites);
869	+ $this->initializeArchiveIdCache($doneFlag);
870	+
871	+ $idArchive = $archiveLoader->prepareArchive($plugin);
872	+
873	+ if ($idArchive) {
874	+ $this->idarchives[$doneFlag][$periodString][] = $idArchive;
875	+ }
876	+ }
877	+ }
878	+
879	+ private function getIdArchivesByMonth($doneFlags)
880	+ {
881	+ // order idarchives by the table month they belong to
882	+ $idArchivesByMonth = array();
883	+
884	+ foreach (array_keys($doneFlags) as $doneFlag) {
885	+ if (empty($this->idarchives[$doneFlag])) {
886	+ continue;
887	+ }
888	+
889	+ foreach ($this->idarchives[$doneFlag] as $dateRange => $idarchives) {
890	+ foreach ($idarchives as $id) {
891	+ $idArchivesByMonth[$dateRange][] = $id;
892	+ }
893	+ }
894	+ }
895	+
896	+ return $idArchivesByMonth;
897	+ }
898	+
899	+ /**
900	+ * @internal
901	+ */
902	+ public static function clearStaticCache()
903	+ {
904	+ self::$cache = null;
905	+ }
906	+ }

app/core/Archive/ArchiveInvalidator.php ADDED Viewed

	@@ -0,0 +1,481 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+
10	+ namespace Piwik\Archive;
11	+
12	+ use Piwik\Archive\ArchiveInvalidator\InvalidationResult;
13	+ use Piwik\ArchiveProcessor\ArchivingStatus;
14	+ use Piwik\CronArchive\SitesToReprocessDistributedList;
15	+ use Piwik\DataAccess\ArchiveTableCreator;
16	+ use Piwik\DataAccess\Model;
17	+ use Piwik\Date;
18	+ use Piwik\CliMulti\Process;
19	+ use Piwik\Option;
20	+ use Piwik\Common;
21	+ use Piwik\Piwik;
22	+ use Piwik\Plugins\CoreAdminHome\Tasks\ArchivesToPurgeDistributedList;
23	+ use Piwik\Plugins\PrivacyManager\PrivacyManager;
24	+ use Piwik\Period;
25	+ use Piwik\Segment;
26	+ use Piwik\SettingsServer;
27	+ use Piwik\Site;
28	+ use Piwik\Tracker\Cache;
29	+
30	+ /**
31	+ * Service that can be used to invalidate archives or add archive references to a list so they will
32	+ * be invalidated later.
33	+ *
34	+ * Archives are put in an "invalidated" state by setting the done flag to `ArchiveWriter::DONE_INVALIDATED`.
35	+ * This class also adds the archive's associated site to the a distributed list and adding the archive's year month to another
36	+ * distributed list.
37	+ *
38	+ * CronArchive will reprocess the archive data for all sites in the first list, and a scheduled task
39	+ * will purge the old, invalidated data in archive tables identified by the second list.
40	+ *
41	+ * Until CronArchive, or browser triggered archiving, re-processes data for an invalidated archive, the invalidated
42	+ * archive data will still be displayed in the UI and API.
43	+ *
44	+ * ### Deferred Invalidation
45	+ *
46	+ * Invalidating archives means running queries on one or more archive tables. In some situations, like during
47	+ * tracking, this is not desired. In such cases, archive references can be added to a list via the
48	+ * rememberToInvalidateArchivedReportsLater method, which will add the reference to a distributed list
49	+ *
50	+ * Later, during Piwik's normal execution, the list will be read and every archive it references will
51	+ * be invalidated.
52	+ */
53	+ class ArchiveInvalidator
54	+ {
55	+ const TRACKER_CACHE_KEY = 'ArchiveInvalidator.rememberToInvalidate';
56	+
57	+ private $rememberArchivedReportIdStart = 'report_to_invalidate_';
58	+
59	+ /**
60	+ * @var Model
61	+ */
62	+ private $model;
63	+
64	+ /**
65	+ * @var ArchivingStatus
66	+ */
67	+ private $archivingStatus;
68	+
69	+ public function __construct(Model $model, ArchivingStatus $archivingStatus)
70	+ {
71	+ $this->model = $model;
72	+ $this->archivingStatus = $archivingStatus;
73	+ }
74	+
75	+ public function getAllRememberToInvalidateArchivedReportsLater()
76	+ {
77	+ // we do not really have to get the value first. we could simply always try to call set() and it would update or
78	+ // insert the record if needed but we do not want to lock the table (especially since there are still some
79	+ // MyISAM installations)
80	+ $values = Option::getLike($this->rememberArchivedReportIdStart . '%');
81	+
82	+ $all = [];
83	+ foreach ($values as $name => $value) {
84	+ $suffix = substr($name, strlen($this->rememberArchivedReportIdStart));
85	+ list($idSite, $dateStr) = explode('_', $suffix);
86	+
87	+ $all[$idSite][$dateStr] = $value;
88	+ }
89	+ return $all;
90	+ }
91	+
92	+ public function rememberToInvalidateArchivedReportsLater($idSite, Date $date)
93	+ {
94	+ if (SettingsServer::isTrackerApiRequest()) {
95	+ $value = $this->getRememberedArchivedReportsOptionFromTracker($idSite, $date->toString());
96	+ } else {
97	+ // To support multiple transactions at once, look for any other process to have set (and committed)
98	+ // this report to be invalidated.
99	+ $key = $this->buildRememberArchivedReportIdForSiteAndDate($idSite, $date->toString());
100	+
101	+ // we do not really have to get the value first. we could simply always try to call set() and it would update or
102	+ // insert the record if needed but we do not want to lock the table (especially since there are still some
103	+ // MyISAM installations)
104	+ $value = Option::getLike($key . '%');
105	+ }
106	+
107	+ // getLike() returns an empty array rather than 'false'
108	+ if (empty($value)) {
109	+ // In order to support multiple concurrent transactions, add our pid to the end of the key so that it will just insert
110	+ // rather than waiting on some other process to commit before proceeding.The issue is that with out this, more than
111	+ // one process is trying to add the exact same value to the table, which causes contention. With the pid suffixed to
112	+ // the value, each process can successfully enter its own row in the table. The net result will be the same. We could
113	+ // always just set this, but it would result in a lot of rows in the options table.. more than needed. With this
114	+ // change you'll have at most N rows per date/site, where N is the number of parallel requests on this same idsite/date
115	+ // that happen to run in overlapping transactions.
116	+ $mykey = $this->buildRememberArchivedReportIdProcessSafe($idSite, $date->toString());
117	+ Option::set($mykey, '1');
118	+ Cache::clearCacheGeneral();
119	+ }
120	+ }
121	+
122	+ private function getRememberedArchivedReportsOptionFromTracker($idSite, $dateStr)
123	+ {
124	+ $cacheKey = self::TRACKER_CACHE_KEY;
125	+
126	+ $generalCache = Cache::getCacheGeneral();
127	+ if (empty($generalCache[$cacheKey][$idSite][$dateStr])) {
128	+ Cache::clearCacheGeneral();
129	+ return [];
130	+ }
131	+
132	+ return $generalCache[$cacheKey][$idSite][$dateStr];
133	+ }
134	+
135	+ public function getRememberedArchivedReportsThatShouldBeInvalidated()
136	+ {
137	+ $reports = Option::getLike($this->rememberArchivedReportIdStart . '%_%');
138	+
139	+ $sitesPerDay = array();
140	+
141	+ foreach ($reports as $report => $value) {
142	+ $report = str_replace($this->rememberArchivedReportIdStart, '', $report);
143	+ $report = explode('_', $report);
144	+ $siteId = (int) $report[0];
145	+ $date = $report[1];
146	+
147	+ if (empty($sitesPerDay[$date])) {
148	+ $sitesPerDay[$date] = array();
149	+ }
150	+
151	+ $sitesPerDay[$date][] = $siteId;
152	+ }
153	+
154	+ return $sitesPerDay;
155	+ }
156	+
157	+ private function buildRememberArchivedReportIdForSite($idSite)
158	+ {
159	+ return $this->rememberArchivedReportIdStart . (int) $idSite;
160	+ }
161	+
162	+ private function buildRememberArchivedReportIdForSiteAndDate($idSite, $date)
163	+ {
164	+ $id = $this->buildRememberArchivedReportIdForSite($idSite);
165	+ $id .= '_' . trim($date);
166	+
167	+ return $id;
168	+ }
169	+
170	+ // This version is multi process safe on the insert of a new date to invalidate.
171	+ private function buildRememberArchivedReportIdProcessSafe($idSite, $date)
172	+ {
173	+ $id = $this->buildRememberArchivedReportIdForSiteAndDate($idSite, $date);
174	+ $id .= '_' . Common::getProcessId();
175	+ return $id;
176	+ }
177	+
178	+ public function forgetRememberedArchivedReportsToInvalidateForSite($idSite)
179	+ {
180	+ $id = $this->buildRememberArchivedReportIdForSite($idSite) . '_%';
181	+ Option::deleteLike($id);
182	+ Cache::clearCacheGeneral();
183	+ }
184	+
185	+ /**
186	+ * @internal
187	+ */
188	+ public function forgetRememberedArchivedReportsToInvalidate($idSite, Date $date)
189	+ {
190	+ $id = $this->buildRememberArchivedReportIdForSiteAndDate($idSite, $date->toString());
191	+
192	+ // The process pid is added to the end of the entry in order to support multiple concurrent transactions.
193	+ // So this must be a deleteLike call to get all the entries, where there used to only be one.
194	+ Option::deleteLike($id . '%');
195	+ Cache::clearCacheGeneral();
196	+ }
197	+
198	+ /**
199	+ * @param $idSites int[]
200	+ * @param $dates Date[]
201	+ * @param $period string
202	+ * @param $segment Segment
203	+ * @param bool $cascadeDown
204	+ * @return InvalidationResult
205	+ * @throws \Exception
206	+ */
207	+ public function markArchivesAsInvalidated(array $idSites, array $dates, $period, Segment $segment = null, $cascadeDown = false)
208	+ {
209	+ $invalidationInfo = new InvalidationResult();
210	+
211	+ // quick fix for #15086, if we're only invalidating today's date for a site, don't add the site to the list of sites
212	+ // to reprocess.
213	+ $hasMoreThanJustToday = [];
214	+ foreach ($idSites as $idSite) {
215	+ $hasMoreThanJustToday[$idSite] = true;
216	+ $tz = Site::getTimezoneFor($idSite);
217	+
218	+ if (($period == 'day' \|\| $period === false)
219	+ && count($dates) == 1
220	+ && $dates[0]->toString() == Date::factoryInTimezone('today', $tz)
221	+ ) {
222	+ $hasMoreThanJustToday[$idSite] = false;
223	+ }
224	+ }
225	+
226	+ /**
227	+ * Triggered when a Matomo user requested the invalidation of some reporting archives. Using this event, plugin
228	+ * developers can automatically invalidate another site, when a site is being invalidated. A plugin may even
229	+ * remove an idSite from the list of sites that should be invalidated to prevent it from ever being
230	+ * invalidated.
231	+ *
232	+ * Example
233	+ *
234	+ * public function getIdSitesToMarkArchivesAsInvalidates(&$idSites)
235	+ * {
236	+ * if (in_array(1, $idSites)) {
237	+ * $idSites[] = 5; // when idSite 1 is being invalidated, also invalidate idSite 5
238	+ * }
239	+ * }
240	+ *
241	+ * @param array &$idSites An array containing a list of site IDs which are requested to be invalidated.
242	+ */
243	+ Piwik::postEvent('Archiving.getIdSitesToMarkArchivesAsInvalidated', array(&$idSites));
244	+ // we trigger above event on purpose here and it is good that the segment was created like
245	+ // `new Segment($segmentString, $idSites)` because when a user adds a site via this event, the added idSite
246	+ // might not have this segment meaning we avoid a possible error. For the workflow to work, any added or removed
247	+ // idSite does not need to be added to $segment.
248	+
249	+ $datesToInvalidate = $this->removeDatesThatHaveBeenPurged($dates, $invalidationInfo);
250	+
251	+ if (empty($period)) {
252	+ // if the period is empty, we don't need to cascade in any way, since we'll remove all periods
253	+ $periodDates = $this->getDatesByYearMonthAndPeriodType($dates);
254	+ } else {
255	+ $periods = $this->getPeriodsToInvalidate($datesToInvalidate, $period, $cascadeDown);
256	+ $periodDates = $this->getPeriodDatesByYearMonthAndPeriodType($periods);
257	+ }
258	+
259	+ $periodDates = $this->getUniqueDates($periodDates);
260	+
261	+ $this->markArchivesInvalidated($idSites, $periodDates, $segment);
262	+
263	+ $yearMonths = array_keys($periodDates);
264	+ $this->markInvalidatedArchivesForReprocessAndPurge($idSites, $yearMonths, $hasMoreThanJustToday);
265	+
266	+ foreach ($idSites as $idSite) {
267	+ foreach ($dates as $date) {
268	+ $this->forgetRememberedArchivedReportsToInvalidate($idSite, $date);
269	+ }
270	+ }
271	+
272	+ return $invalidationInfo;
273	+ }
274	+
275	+ /**
276	+ * @param $idSites int[]
277	+ * @param $dates Date[]
278	+ * @param $period string
279	+ * @param $segment Segment
280	+ * @param bool $cascadeDown
281	+ * @return InvalidationResult
282	+ * @throws \Exception
283	+ */
284	+ public function markArchivesOverlappingRangeAsInvalidated(array $idSites, array $dates, Segment $segment = null)
285	+ {
286	+ $invalidationInfo = new InvalidationResult();
287	+
288	+ $ranges = array();
289	+ foreach ($dates as $dateRange) {
290	+ $ranges[] = $dateRange[0] . ',' . $dateRange[1];
291	+ }
292	+ $periodsByType = array(Period\Range::PERIOD_ID => $ranges);
293	+
294	+ $invalidatedMonths = array();
295	+ $archiveNumericTables = ArchiveTableCreator::getTablesArchivesInstalled($type = ArchiveTableCreator::NUMERIC_TABLE);
296	+ foreach ($archiveNumericTables as $table) {
297	+ $tableDate = ArchiveTableCreator::getDateFromTableName($table);
298	+
299	+ $result = $this->model->updateArchiveAsInvalidated($table, $idSites, $periodsByType, $segment);
300	+ $rowsAffected = $result->rowCount();
301	+ if ($rowsAffected > 0) {
302	+ $invalidatedMonths[] = $tableDate;
303	+ }
304	+ }
305	+
306	+ foreach ($idSites as $idSite) {
307	+ foreach ($dates as $dateRange) {
308	+ $this->forgetRememberedArchivedReportsToInvalidate($idSite, $dateRange[0]);
309	+ $invalidationInfo->processedDates[] = $dateRange[0];
310	+ }
311	+ }
312	+
313	+ $archivesToPurge = new ArchivesToPurgeDistributedList();
314	+ $archivesToPurge->add($invalidatedMonths);
315	+
316	+ return $invalidationInfo;
317	+ }
318	+
319	+ /**
320	+ * @param string[][][] $periodDates
321	+ * @return string[][][]
322	+ */
323	+ private function getUniqueDates($periodDates)
324	+ {
325	+ $result = array();
326	+ foreach ($periodDates as $yearMonth => $periodsByYearMonth) {
327	+ foreach ($periodsByYearMonth as $periodType => $periods) {
328	+ $result[$yearMonth][$periodType] = array_unique($periods);
329	+ }
330	+ }
331	+ return $result;
332	+ }
333	+
334	+ /**
335	+ * @param Date[] $dates
336	+ * @param string $periodType
337	+ * @param bool $cascadeDown
338	+ * @return Period[]
339	+ */
340	+ private function getPeriodsToInvalidate($dates, $periodType, $cascadeDown)
341	+ {
342	+ $periodsToInvalidate = array();
343	+
344	+ if ($periodType == 'range') {
345	+ $rangeString = $dates[0] . ',' . $dates[1];
346	+ $periodsToInvalidate[] = Period\Factory::build('range', $rangeString);
347	+ return $periodsToInvalidate;
348	+ }
349	+
350	+ foreach ($dates as $date) {
351	+ $period = Period\Factory::build($periodType, $date);
352	+ $periodsToInvalidate[] = $period;
353	+
354	+ if ($cascadeDown) {
355	+ $periodsToInvalidate = array_merge($periodsToInvalidate, $period->getAllOverlappingChildPeriods());
356	+ }
357	+
358	+ if ($periodType != 'year') {
359	+ $periodsToInvalidate[] = Period\Factory::build('year', $date);
360	+ }
361	+ }
362	+
363	+ return $periodsToInvalidate;
364	+ }
365	+
366	+ /**
367	+ * @param Period[] $periods
368	+ * @return string[][][]
369	+ */
370	+ private function getPeriodDatesByYearMonthAndPeriodType($periods)
371	+ {
372	+ $result = array();
373	+ foreach ($periods as $period) {
374	+ $date = $period->getDateStart();
375	+ $periodType = $period->getId();
376	+
377	+ $yearMonth = ArchiveTableCreator::getTableMonthFromDate($date);
378	+ $dateString = $date->toString();
379	+ if ($periodType == Period\Range::PERIOD_ID) {
380	+ $dateString = $period->getRangeString();
381	+ }
382	+ $result[$yearMonth][$periodType][] = $dateString;
383	+ }
384	+ return $result;
385	+ }
386	+
387	+ /**
388	+ * Called when deleting all periods.
389	+ *
390	+ * @param Date[] $dates
391	+ * @return string[][][]
392	+ */
393	+ private function getDatesByYearMonthAndPeriodType($dates)
394	+ {
395	+ $result = array();
396	+ foreach ($dates as $date) {
397	+ $yearMonth = ArchiveTableCreator::getTableMonthFromDate($date);
398	+ $result[$yearMonth][null][] = $date->toString();
399	+
400	+ // since we're removing all periods, we must make sure to remove year periods as well.
401	+ // this means we have to make sure the january table is processed.
402	+ $janYearMonth = $date->toString('Y') . '_01';
403	+ $result[$janYearMonth][null][] = $date->toString();
404	+ }
405	+ return $result;
406	+ }
407	+
408	+ /**
409	+ * @param int[] $idSites
410	+ * @param string[][][] $dates
411	+ * @throws \Exception
412	+ */
413	+ private function markArchivesInvalidated($idSites, $dates, Segment $segment = null)
414	+ {
415	+ $archiveNumericTables = ArchiveTableCreator::getTablesArchivesInstalled($type = ArchiveTableCreator::NUMERIC_TABLE);
416	+ foreach ($archiveNumericTables as $table) {
417	+ $tableDate = ArchiveTableCreator::getDateFromTableName($table);
418	+ if (empty($dates[$tableDate])) {
419	+ continue;
420	+ }
421	+
422	+ $this->model->updateArchiveAsInvalidated($table, $idSites, $dates[$tableDate], $segment);
423	+ }
424	+ }
425	+
426	+ /**
427	+ * @param Date[] $dates
428	+ * @param InvalidationResult $invalidationInfo
429	+ * @return \Piwik\Date[]
430	+ */
431	+ private function removeDatesThatHaveBeenPurged($dates, InvalidationResult $invalidationInfo)
432	+ {
433	+ $this->findOlderDateWithLogs($invalidationInfo);
434	+
435	+ $result = array();
436	+ foreach ($dates as $date) {
437	+ // we should only delete reports for dates that are more recent than N days
438	+ if ($invalidationInfo->minimumDateWithLogs
439	+ && $date->isEarlier($invalidationInfo->minimumDateWithLogs)
440	+ ) {
441	+ $invalidationInfo->warningDates[] = $date->toString();
442	+ continue;
443	+ }
444	+
445	+ $result[] = $date;
446	+ $invalidationInfo->processedDates[] = $date->toString();
447	+ }
448	+ return $result;
449	+ }
450	+
451	+ private function findOlderDateWithLogs(InvalidationResult $info)
452	+ {
453	+ // If using the feature "Delete logs older than N days"...
454	+ $purgeDataSettings = PrivacyManager::getPurgeDataSettings();
455	+ $logsDeletedWhenOlderThanDays = (int)$purgeDataSettings['delete_logs_older_than'];
456	+ $logsDeleteEnabled = $purgeDataSettings['delete_logs_enable'];
457	+
458	+ if ($logsDeleteEnabled
459	+ && $logsDeletedWhenOlderThanDays
460	+ ) {
461	+ $info->minimumDateWithLogs = Date::factory('today')->subDay($logsDeletedWhenOlderThanDays);
462	+ }
463	+ }
464	+
465	+ /**
466	+ * @param array $idSites
467	+ * @param array $yearMonths
468	+ */
469	+ private function markInvalidatedArchivesForReprocessAndPurge(array $idSites, $yearMonths, $hasMoreThanJustToday)
470	+ {
471	+ $store = new SitesToReprocessDistributedList();
472	+ foreach ($idSites as $idSite) {
473	+ if (!empty($hasMoreThanJustToday[$idSite])) {
474	+ $store->add($idSite);
475	+ }
476	+ }
477	+
478	+ $archivesToPurge = new ArchivesToPurgeDistributedList();
479	+ $archivesToPurge->add($yearMonths);
480	+ }
481	+ }

app/core/Archive/ArchiveInvalidator/InvalidationResult.php ADDED Viewed

	@@ -0,0 +1,56 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ */
8	+
9	+ namespace Piwik\Archive\ArchiveInvalidator;
10	+
11	+ use Piwik\Date;
12	+
13	+ /**
14	+ * Information about the result of an archive invalidation operation.
15	+ */
16	+ class InvalidationResult
17	+ {
18	+ /**
19	+ * Dates that couldn't be invalidated because they are earlier than the configured log
20	+ * deletion limit.
21	+ *
22	+ * @var array
23	+ */
24	+ public $warningDates = array();
25	+
26	+ /**
27	+ * Dates that were successfully invalidated.
28	+ *
29	+ * @var array
30	+ */
31	+ public $processedDates = array();
32	+
33	+ /**
34	+ * The day of the oldest log entry.
35	+ *
36	+ * @var Date\|bool
37	+ */
38	+ public $minimumDateWithLogs = false;
39	+
40	+ /**
41	+ * @return string[]
42	+ */
43	+ public function makeOutputLogs()
44	+ {
45	+ $output = array();
46	+ if ($this->warningDates) {
47	+ $output[] = 'Warning: the following Dates have not been invalidated, because they are earlier than your Log Deletion limit: ' .
48	+ implode(", ", $this->warningDates) .
49	+ "\n The last day with logs is " . $this->minimumDateWithLogs . ". " .
50	+ "\n Please disable 'Delete old Logs' or set it to a higher deletion threshold (eg. 180 days or 365 years).'.";
51	+ }
52	+
53	+ $output[] = "Success. The following dates were invalidated successfully: " . implode(", ", $this->processedDates);
54	+ return $output;
55	+ }
56	+ }

app/core/Archive/ArchivePurger.php ADDED Viewed

	@@ -0,0 +1,338 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik\Archive;
10	+
11	+ use Piwik\ArchiveProcessor\Rules;
12	+ use Piwik\Common;
13	+ use Piwik\Config;
14	+ use Piwik\Container\StaticContainer;
15	+ use Piwik\DataAccess\ArchiveTableCreator;
16	+ use Piwik\DataAccess\Model;
17	+ use Piwik\Date;
18	+ use Piwik\Db;
19	+ use Piwik\Piwik;
20	+ use Psr\Log\LoggerInterface;
21	+ use Psr\Log\LogLevel;
22	+
23	+ /**
24	+ * Service that purges temporary, error-ed, invalid and custom range archives from archive tables.
25	+ *
26	+ * Temporary archives are purged if they were archived before a specific time. The time is dependent
27	+ * on whether browser triggered archiving is enabled or not.
28	+ *
29	+ * Error-ed archives are purged w/o constraint.
30	+ *
31	+ * Invalid archives are purged if a new, valid, archive exists w/ the same site, date, period combination.
32	+ * Archives are marked as invalid via Piwik\Archive\ArchiveInvalidator.
33	+ */
34	+ class ArchivePurger
35	+ {
36	+ /**
37	+ * @var Model
38	+ */
39	+ private $model;
40	+
41	+ /**
42	+ * Date threshold for purging custom range archives. Archives that are older than this date
43	+ * are purged unconditionally from the requested archive table.
44	+ *
45	+ * @var Date
46	+ */
47	+ private $purgeCustomRangesOlderThan;
48	+
49	+ /**
50	+ * Date to use for 'yesterday'. Exists so tests can override this value.
51	+ *
52	+ * @var Date
53	+ */
54	+ private $yesterday;
55	+
56	+ /**
57	+ * Date to use for 'today'. Exists so tests can override this value.
58	+ *
59	+ * @var $today
60	+ */
61	+ private $today;
62	+
63	+ /**
64	+ * Date to use for 'now'. Exists so tests can override this value.
65	+ *
66	+ * @var int
67	+ */
68	+ private $now;
69	+
70	+ /**
71	+ * @var LoggerInterface
72	+ */
73	+ private $logger;
74	+
75	+ public function __construct(Model $model = null, Date $purgeCustomRangesOlderThan = null, LoggerInterface $logger = null)
76	+ {
77	+ $this->model = $model ?: new Model();
78	+
79	+ $this->purgeCustomRangesOlderThan = $purgeCustomRangesOlderThan ?: self::getDefaultCustomRangeToPurgeAgeThreshold();
80	+
81	+ $this->yesterday = Date::factory('yesterday');
82	+ $this->today = Date::factory('today');
83	+ $this->now = time();
84	+ $this->logger = $logger ?: StaticContainer::get('Psr\Log\LoggerInterface');
85	+ }
86	+
87	+ /**
88	+ * Purge all invalidate archives for whom there are newer, valid archives from the archive
89	+ * table that stores data for `$date`.
90	+ *
91	+ * @param Date $date The date identifying the archive table.
92	+ * @return int The total number of archive rows deleted (from both the blog & numeric tables).
93	+ */
94	+ public function purgeInvalidatedArchivesFrom(Date $date)
95	+ {
96	+ $numericTable = ArchiveTableCreator::getNumericTable($date);
97	+
98	+ // we don't want to do an INNER JOIN on every row in a archive table that can potentially have tens to hundreds of thousands of rows,
99	+ // so we first look for sites w/ invalidated archives, and use this as a constraint in getInvalidatedArchiveIdsSafeToDelete() below.
100	+ // the constraint will hit an INDEX and speed up the inner join that happens in getInvalidatedArchiveIdsSafeToDelete().
101	+ $idSites = $this->model->getSitesWithInvalidatedArchive($numericTable);
102	+ if (empty($idSites)) {
103	+ $this->logger->debug("No sites with invalidated archives found in {table}.", array('table' => $numericTable));
104	+ return 0;
105	+ }
106	+
107	+ $archiveIds = $this->model->getInvalidatedArchiveIdsSafeToDelete($numericTable, $idSites);
108	+ if (empty($archiveIds)) {
109	+ $this->logger->debug("No invalidated archives found in {table} with newer, valid archives.", array('table' => $numericTable));
110	+ return 0;
111	+ }
112	+
113	+ $this->logger->info("Found {countArchiveIds} invalidated archives safe to delete in {table}.", array(
114	+ 'table' => $numericTable, 'countArchiveIds' => count($archiveIds)
115	+ ));
116	+
117	+ $deletedRowCount = $this->deleteArchiveIds($date, $archiveIds);
118	+
119	+ $this->logger->debug("Deleted {count} rows in {table} and its associated blob table.", array(
120	+ 'table' => $numericTable, 'count' => $deletedRowCount
121	+ ));
122	+
123	+ return $deletedRowCount;
124	+ }
125	+
126	+ /**
127	+ * Removes the outdated archives for the given month.
128	+ * (meaning they are marked with a done flag of ArchiveWriter::DONE_OK_TEMPORARY or ArchiveWriter::DONE_ERROR)
129	+ *
130	+ * @param Date $dateStart Only the month will be used
131	+ * @return int Returns the total number of rows deleted.
132	+ */
133	+ public function purgeOutdatedArchives(Date $dateStart)
134	+ {
135	+ $purgeArchivesOlderThan = $this->getOldestTemporaryArchiveToKeepThreshold();
136	+ $deletedRowCount = 0;
137	+
138	+ $idArchivesToDelete = $this->getOutdatedArchiveIds($dateStart, $purgeArchivesOlderThan);
139	+ if (!empty($idArchivesToDelete)) {
140	+ $deletedRowCount = $this->deleteArchiveIds($dateStart, $idArchivesToDelete);
141	+
142	+ $this->logger->info("Deleted {count} rows in archive tables (numeric + blob) for {date}.", array(
143	+ 'count' => $deletedRowCount,
144	+ 'date' => $dateStart
145	+ ));
146	+ } else {
147	+ $this->logger->debug("No outdated archives found in archive numeric table for {date}.", array('date' => $dateStart));
148	+ }
149	+
150	+ $this->logger->debug("Purging temporary archives: done [ purged archives older than {date} in {yearMonth} ] [Deleted IDs: {deletedIds}]", array(
151	+ 'date' => $purgeArchivesOlderThan,
152	+ 'yearMonth' => $dateStart->toString('Y-m'),
153	+ 'deletedIds' => implode(',', $idArchivesToDelete)
154	+ ));
155	+
156	+ return $deletedRowCount;
157	+ }
158	+
159	+ public function purgeDeletedSiteArchives(Date $dateStart)
160	+ {
161	+ $archiveTable = ArchiveTableCreator::getNumericTable($dateStart);
162	+ $idArchivesToDelete = $this->model->getArchiveIdsForDeletedSites($archiveTable);
163	+
164	+ return $this->purge($idArchivesToDelete, $dateStart, 'deleted sites');
165	+ }
166	+
167	+ /**
168	+ * @param Date $dateStart
169	+ * @param array $deletedSegments List of segments whose archives should be purged
170	+ * @return int
171	+ */
172	+ public function purgeDeletedSegmentArchives(Date $dateStart, array $deletedSegments)
173	+ {
174	+ if (count($deletedSegments)) {
175	+ $idArchivesToDelete = $this->getDeletedSegmentArchiveIds($dateStart, $deletedSegments);
176	+ return $this->purge($idArchivesToDelete, $dateStart, 'deleted segments');
177	+ }
178	+ }
179	+
180	+ /**
181	+ * Purge all numeric and blob archives with the given IDs from the database.
182	+ * @param array $idArchivesToDelete
183	+ * @param Date $dateStart
184	+ * @param string $reason
185	+ * @return int
186	+ */
187	+ protected function purge(array $idArchivesToDelete, Date $dateStart, $reason)
188	+ {
189	+ $deletedRowCount = 0;
190	+ if (!empty($idArchivesToDelete)) {
191	+ $deletedRowCount = $this->deleteArchiveIds($dateStart, $idArchivesToDelete);
192	+
193	+ $this->logger->info(
194	+ "Deleted {count} rows in archive tables (numeric + blob) for {reason} for {date}.",
195	+ array(
196	+ 'count' => $deletedRowCount,
197	+ 'date' => $dateStart,
198	+ 'reason' => $reason
199	+ )
200	+ );
201	+
202	+ $this->logger->debug("[Deleted IDs: {deletedIds}]", array(
203	+ 'deletedIds' => implode(',', $idArchivesToDelete)
204	+ ));
205	+ } else {
206	+ $this->logger->debug(
207	+ "No archives for {reason} found in archive numeric table for {date}.",
208	+ array('date' => $dateStart, 'reason' => $reason)
209	+ );
210	+ }
211	+
212	+ return $deletedRowCount;
213	+ }
214	+
215	+ protected function getDeletedSegmentArchiveIds(Date $date, array $deletedSegments)
216	+ {
217	+ $archiveTable = ArchiveTableCreator::getNumericTable($date);
218	+ return $this->model->getArchiveIdsForSegments(
219	+ $archiveTable, $deletedSegments, $this->getOldestTemporaryArchiveToKeepThreshold()
220	+ );
221	+ }
222	+
223	+ protected function getOutdatedArchiveIds(Date $date, $purgeArchivesOlderThan)
224	+ {
225	+ $archiveTable = ArchiveTableCreator::getNumericTable($date);
226	+
227	+ $result = $this->model->getTemporaryArchivesOlderThan($archiveTable, $purgeArchivesOlderThan);
228	+
229	+ $idArchivesToDelete = array();
230	+ if (!empty($result)) {
231	+ foreach ($result as $row) {
232	+ $idArchivesToDelete[] = $row['idarchive'];
233	+ }
234	+ }
235	+
236	+ return $idArchivesToDelete;
237	+ }
238	+
239	+ /**
240	+ * Deleting "Custom Date Range" reports after 1 day, since they can be re-processed and would take up un-necessary space.
241	+ *
242	+ * @param $date Date
243	+ * @return int The total number of rows deleted from both the numeric & blob table.
244	+ */
245	+ public function purgeArchivesWithPeriodRange(Date $date)
246	+ {
247	+ $numericTable = ArchiveTableCreator::getNumericTable($date);
248	+ $blobTable = ArchiveTableCreator::getBlobTable($date);
249	+
250	+ $deletedCount = $this->model->deleteArchivesWithPeriod(
251	+ $numericTable, $blobTable, Piwik::$idPeriods['range'], $this->purgeCustomRangesOlderThan);
252	+
253	+ $level = $deletedCount == 0 ? LogLevel::DEBUG : LogLevel::INFO;
254	+ $this->logger->log($level, "Purged {count} range archive rows from {numericTable} & {blobTable}.", array(
255	+ 'count' => $deletedCount,
256	+ 'numericTable' => $numericTable,
257	+ 'blobTable' => $blobTable
258	+ ));
259	+
260	+ $this->logger->debug(" [ purged archives older than {threshold} ]", array('threshold' => $this->purgeCustomRangesOlderThan));
261	+
262	+ return $deletedCount;
263	+ }
264	+
265	+ /**
266	+ * Deletes by batches Archive IDs in the specified month,
267	+ *
268	+ * @param Date $date
269	+ * @param $idArchivesToDelete
270	+ * @return int Number of rows deleted from both numeric + blob table.
271	+ */
272	+ protected function deleteArchiveIds(Date $date, $idArchivesToDelete)
273	+ {
274	+ $batches = array_chunk($idArchivesToDelete, 1000);
275	+ $numericTable = ArchiveTableCreator::getNumericTable($date);
276	+ $blobTable = ArchiveTableCreator::getBlobTable($date);
277	+
278	+ $deletedCount = 0;
279	+ foreach ($batches as $idsToDelete) {
280	+ $deletedCount += $this->model->deleteArchiveIds($numericTable, $blobTable, $idsToDelete);
281	+ }
282	+ return $deletedCount;
283	+ }
284	+
285	+ /**
286	+ * Returns a timestamp indicating outdated archives older than this timestamp (processed before) can be purged.
287	+ *
288	+ * @return int\|bool Outdated archives older than this timestamp should be purged
289	+ */
290	+ protected function getOldestTemporaryArchiveToKeepThreshold()
291	+ {
292	+ $temporaryArchivingTimeout = Rules::getTodayArchiveTimeToLive();
293	+ if (Rules::isBrowserTriggerEnabled()) {
294	+ // If Browser Archiving is enabled, it is likely there are many more temporary archives
295	+ // We delete more often which is safe, since reports are re-processed on demand
296	+ return Date::factory($this->now - 2 * $temporaryArchivingTimeout)->getDateTime();
297	+ }
298	+
299	+ // If cron core:archive command is building the reports, we should keep all temporary reports from today
300	+ return $this->yesterday->getDateTime();
301	+ }
302	+
303	+ private static function getDefaultCustomRangeToPurgeAgeThreshold()
304	+ {
305	+ $daysRangesValid = Config::getInstance()->General['purge_date_range_archives_after_X_days'];
306	+ return Date::factory('today')->subDay($daysRangesValid)->getDateTime();
307	+ }
308	+
309	+ /**
310	+ * For tests.
311	+ *
312	+ * @param Date $yesterday
313	+ */
314	+ public function setYesterdayDate(Date $yesterday)
315	+ {
316	+ $this->yesterday = $yesterday;
317	+ }
318	+
319	+ /**
320	+ * For tests.
321	+ *
322	+ * @param Date $today
323	+ */
324	+ public function setTodayDate(Date $today)
325	+ {
326	+ $this->today = $today;
327	+ }
328	+
329	+ /**
330	+ * For tests.
331	+ *
332	+ * @param int $now
333	+ */
334	+ public function setNow($now)
335	+ {
336	+ $this->now = $now;
337	+ }
338	+ }

app/core/Archive/ArchiveQuery.php ADDED Viewed

	@@ -0,0 +1,49 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik\Archive;
10	+
11	+
12	+ use Piwik\DataTable;
13	+
14	+ interface ArchiveQuery
15	+ {
16	+ /**
17	+ * @param string\|string[] $names
18	+ * @return false\|number\|array
19	+ */
20	+ public function getNumeric($names);
21	+
22	+ /**
23	+ * @param string\|string[] $names
24	+ * @return DataTable\|DataTable\Map
25	+ */
26	+ public function getDataTableFromNumeric($names);
27	+
28	+ /**
29	+ * @param $names
30	+ * @return mixed
31	+ */
32	+ public function getDataTableFromNumericAndMergeChildren($names);
33	+
34	+ /**
35	+ * @param string $name
36	+ * @param int\|string\|null $idSubtable
37	+ * @return DataTable\|DataTable\Map
38	+ */
39	+ public function getDataTable($name, $idSubtable = null);
40	+
41	+ /**
42	+ * @param string $name
43	+ * @param int\|string\|null $idSubtable
44	+ * @param int\|null $depth
45	+ * @param bool $addMetadataSubtableId
46	+ * @return DataTable\|DataTable\Map
47	+ */
48	+ public function getDataTableExpanded($name, $idSubtable = null, $depth = null, $addMetadataSubtableId = true);
49	+ }

app/core/Archive/ArchiveQueryFactory.php ADDED Viewed

	@@ -0,0 +1,127 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+
10	+ namespace Piwik\Archive;
11	+
12	+ use Piwik\Archive;
13	+ use Piwik\Period;
14	+ use Piwik\Segment;
15	+ use Piwik\Site;
16	+ use Piwik\Period\Factory as PeriodFactory;
17	+
18	+ class ArchiveQueryFactory
19	+ {
20	+ public function __construct()
21	+ {
22	+ // empty
23	+ }
24	+
25	+ /**
26	+ * @see \Piwik\Archive::build()
27	+ */
28	+ public function build($idSites, $strPeriod, $strDate, $strSegment = false, $_restrictSitesToLogin = false)
29	+ {
30	+ list($websiteIds, $timezone, $idSiteIsAll) = $this->getSiteInfoFromQueryParam($idSites, $_restrictSitesToLogin);
31	+ list($allPeriods, $isMultipleDate) = $this->getPeriodInfoFromQueryParam($strDate, $strPeriod, $timezone);
32	+ $segment = $this->getSegmentFromQueryParam($strSegment, $websiteIds);
33	+
34	+ return $this->factory($segment, $allPeriods, $websiteIds, $idSiteIsAll, $isMultipleDate);
35	+ }
36	+
37	+ /**
38	+ * @see \Piwik\Archive::factory()
39	+ */
40	+ public function factory(Segment $segment, array $periods, array $idSites, $idSiteIsAll = false, $isMultipleDate = false)
41	+ {
42	+ $forceIndexedBySite = false;
43	+ $forceIndexedByDate = false;
44	+
45	+ if ($idSiteIsAll \|\| count($idSites) > 1) {
46	+ $forceIndexedBySite = true;
47	+ }
48	+
49	+ if (count($periods) > 1 \|\| $isMultipleDate) {
50	+ $forceIndexedByDate = true;
51	+ }
52	+
53	+ $params = new Parameters($idSites, $periods, $segment);
54	+
55	+ return $this->newInstance($params, $forceIndexedBySite, $forceIndexedByDate);
56	+ }
57	+
58	+ public function newInstance(Parameters $params, $forceIndexedBySite, $forceIndexedByDate)
59	+ {
60	+ return new Archive($params, $forceIndexedBySite, $forceIndexedByDate);
61	+ }
62	+
63	+ /**
64	+ * Parses the site ID string provided in the 'idSite' query parameter to a list of
65	+ * website IDs.
66	+ *
67	+ * @param string $idSites the value of the 'idSite' query parameter
68	+ * @param bool $_restrictSitesToLogin
69	+ * @return array an array containing three elements:
70	+ * - an array of website IDs
71	+ * - string timezone to use (or false to use no timezone) when creating periods.
72	+ * - true if the request was for all websites (this forces the archive result to
73	+ * be indexed by site, even if there is only one site in Piwik)
74	+ */
75	+ protected function getSiteInfoFromQueryParam($idSites, $_restrictSitesToLogin)
76	+ {
77	+ $websiteIds = Site::getIdSitesFromIdSitesString($idSites, $_restrictSitesToLogin);
78	+
79	+ $timezone = false;
80	+ if (count($websiteIds) == 1) {
81	+ $timezone = Site::getTimezoneFor($websiteIds[0]);
82	+ }
83	+
84	+ $idSiteIsAll = $idSites == Archive::REQUEST_ALL_WEBSITES_FLAG;
85	+
86	+ return [$websiteIds, $timezone, $idSiteIsAll];
87	+ }
88	+
89	+ /**
90	+ * Parses the date & period query parameters into a list of periods.
91	+ *
92	+ * @param string $strDate the value of the 'date' query parameter
93	+ * @param string $strPeriod the value of the 'period' query parameter
94	+ * @param string $timezone the timezone to use when constructing periods.
95	+ * @return array an array containing two elements:
96	+ * - the list of period objects to query archive data for
97	+ * - true if the request was for multiple periods (ie, two months, two weeks, etc.), false if otherwise.
98	+ * (this forces the archive result to be indexed by period, even if the list of periods
99	+ * has only one period).
100	+ */
101	+ protected function getPeriodInfoFromQueryParam($strDate, $strPeriod, $timezone)
102	+ {
103	+ if (Period::isMultiplePeriod($strDate, $strPeriod)) {
104	+ $oPeriod = PeriodFactory::build($strPeriod, $strDate, $timezone);
105	+ $allPeriods = $oPeriod->getSubperiods();
106	+ } else {
107	+ $oPeriod = PeriodFactory::makePeriodFromQueryParams($timezone, $strPeriod, $strDate);
108	+ $allPeriods = array($oPeriod);
109	+ }
110	+
111	+ $isMultipleDate = Period::isMultiplePeriod($strDate, $strPeriod);
112	+
113	+ return [$allPeriods, $isMultipleDate];
114	+ }
115	+
116	+ /**
117	+ * Parses the segment query parameter into a Segment object.
118	+ *
119	+ * @param string $strSegment the value of the 'segment' query parameter.
120	+ * @param int[] $websiteIds the list of sites being queried.
121	+ * @return Segment
122	+ */
123	+ protected function getSegmentFromQueryParam($strSegment, $websiteIds)
124	+ {
125	+ return new Segment($strSegment, $websiteIds);
126	+ }
127	+ }

app/core/Archive/Chunk.php ADDED Viewed

	@@ -0,0 +1,144 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+
10	+ namespace Piwik\Archive;
11	+
12	+ use Piwik\DataTable;
13	+
14	+ /**
15	+ * This class is used to split blobs of DataTables into chunks. Each blob used to be stored under one blob in the
16	+ * archive table. For better efficiency we do now combine multiple DataTable into one blob entry.
17	+ *
18	+ * Chunks are identified by having the recordName $recordName_chunk_0_99, $recordName_chunk_100_199 (this chunk stores
19	+ * the subtable 100-199).
20	+ */
21	+ class Chunk
22	+ {
23	+ const ARCHIVE_APPENDIX_SUBTABLES = 'chunk';
24	+ const NUM_TABLES_IN_CHUNK = 100;
25	+
26	+ /**
27	+ * Get's the record name to use for a given tableId/subtableId.
28	+ *
29	+ * @param string $recordName eg 'Actions_ActionsUrl'
30	+ * @param int $tableId eg '5' for tableId '5'
31	+ * @return string eg 'Actions_ActionsUrl_chunk_0_99' as the table should be stored under this blob id.
32	+ */
33	+ public function getRecordNameForTableId($recordName, $tableId)
34	+ {
35	+ $chunk = (floor($tableId / self::NUM_TABLES_IN_CHUNK));
36	+ $start = $chunk * self::NUM_TABLES_IN_CHUNK;
37	+ $end = $start + self::NUM_TABLES_IN_CHUNK - 1;
38	+
39	+ return $recordName . $this->getAppendix() . $start . '_' . $end;
40	+ }
41	+
42	+ /**
43	+ * Moves the given blobs into chunks and assigns a proper record name containing the chunk number.
44	+ *
45	+ * @param string $recordName The original archive record name, eg 'Actions_ActionsUrl'
46	+ * @param array $blobs An array containg a mapping of tableIds to blobs. Eg array(0 => 'blob', 1 => 'subtableBlob', ...)
47	+ * @return array An array where each blob is moved into a chunk, indexed by recordNames.
48	+ * eg array('Actions_ActionsUrl_chunk_0_99' => array(0 => 'blob', 1 => 'subtableBlob', ...),
49	+ * 'Actions_ActionsUrl_chunk_100_199' => array(...))
50	+ */
51	+ public function moveArchiveBlobsIntoChunks($recordName, $blobs)
52	+ {
53	+ $chunks = array();
54	+
55	+ foreach ($blobs as $tableId => $blob) {
56	+ $name = $this->getRecordNameForTableId($recordName, $tableId);
57	+
58	+ if (!array_key_exists($name, $chunks)) {
59	+ $chunks[$name] = array();
60	+ }
61	+
62	+ $chunks[$name][$tableId] = $blob;
63	+ }
64	+
65	+ return $chunks;
66	+ }
67	+
68	+ /**
69	+ * Detects whether a recordName like 'Actions_ActionUrls_chunk_0_99' or 'Actions_ActionUrls' belongs to a
70	+ * chunk or not.
71	+ *
72	+ * To be a valid recordName that belongs to a chunk it must end with '_chunk_NUMERIC_NUMERIC'.
73	+ *
74	+ * @param string $recordName
75	+ * @return bool
76	+ */
77	+ public function isRecordNameAChunk($recordName)
78	+ {
79	+ $posAppendix = $this->getEndPosOfChunkAppendix($recordName);
80	+
81	+ if (false === $posAppendix) {
82	+ return false;
83	+ }
84	+
85	+ // will contain "0_99" of "chunk_0_99"
86	+ $blobId = substr($recordName, $posAppendix);
87	+
88	+ return $this->isChunkRange($blobId);
89	+ }
90	+
91	+ private function isChunkRange($blobId)
92	+ {
93	+ $blobId = explode('_', $blobId);
94	+
95	+ return 2 === count($blobId) && is_numeric($blobId[0]) && is_numeric($blobId[1]);
96	+ }
97	+
98	+ /**
99	+ * When having a record like 'Actions_ActionUrls_chunk_0_99" it will return the raw recordName 'Actions_ActionUrls'.
100	+ *
101	+ * @param string $recordName
102	+ * @return string
103	+ */
104	+ public function getRecordNameWithoutChunkAppendix($recordName)
105	+ {
106	+ if (!$this->isRecordNameAChunk($recordName)) {
107	+ return $recordName;
108	+ }
109	+
110	+ $posAppendix = $this->getStartPosOfChunkAppendix($recordName);
111	+
112	+ if (false === $posAppendix) {
113	+ return $recordName;
114	+ }
115	+
116	+ return substr($recordName, 0, $posAppendix);
117	+ }
118	+
119	+ /**
120	+ * Returns the string that is appended to the original record name. This appendix identifes a record name is a
121	+ * chunk.
122	+ * @return string
123	+ */
124	+ public function getAppendix()
125	+ {
126	+ return '_' . self::ARCHIVE_APPENDIX_SUBTABLES . '_';
127	+ }
128	+
129	+ private function getStartPosOfChunkAppendix($recordName)
130	+ {
131	+ return strpos($recordName, $this->getAppendix());
132	+ }
133	+
134	+ private function getEndPosOfChunkAppendix($recordName)
135	+ {
136	+ $pos = strpos($recordName, $this->getAppendix());
137	+
138	+ if ($pos === false) {
139	+ return false;
140	+ }
141	+
142	+ return $pos + strlen($this->getAppendix());
143	+ }
144	+ }

app/core/Archive/DataCollection.php ADDED Viewed

	@@ -0,0 +1,377 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+
10	+ namespace Piwik\Archive;
11	+
12	+ use Exception;
13	+ use Piwik\DataTable;
14	+
15	+ /**
16	+ * This class is used to hold and transform archive data for the Archive class.
17	+ *
18	+ * Archive data is loaded into an instance of this type, can be indexed by archive
19	+ * metadata (such as the site ID, period string, etc.), and can be transformed into
20	+ * DataTable and Map instances.
21	+ */
22	+ class DataCollection
23	+ {
24	+ const METADATA_CONTAINER_ROW_KEY = '_metadata';
25	+
26	+ /**
27	+ * The archive data, indexed first by site ID and then by period date range. Eg,
28	+ *
29	+ * array(
30	+ * '0' => array(
31	+ * array(
32	+ * '2012-01-01,2012-01-01' => array(...),
33	+ * '2012-01-02,2012-01-02' => array(...),
34	+ * )
35	+ * ),
36	+ * '1' => array(
37	+ * array(
38	+ * '2012-01-01,2012-01-01' => array(...),
39	+ * )
40	+ * )
41	+ * )
42	+ *
43	+ * Archive data can be either a numeric value or a serialized string blob. Every
44	+ * piece of archive data is associated by it's archive name. For example,
45	+ * the array(...) above could look like:
46	+ *
47	+ * array(
48	+ * 'nb_visits' => 1,
49	+ * 'nb_actions' => 2
50	+ * )
51	+ *
52	+ * There is a special element '_metadata' in data rows that holds values treated
53	+ * as DataTable metadata.
54	+ */
55	+ private $data = array();
56	+
57	+ /**
58	+ * The whole list of metric/record names that were used in the archive query.
59	+ *
60	+ * @var array
61	+ */
62	+ private $dataNames;
63	+
64	+ /**
65	+ * The type of data that was queried for (ie, "blob" or "numeric").
66	+ *
67	+ * @var string
68	+ */
69	+ private $dataType;
70	+
71	+ /**
72	+ * The default values to use for each metric/record name that's being queried
73	+ * for.
74	+ *
75	+ * @var array
76	+ */
77	+ private $defaultRow;
78	+
79	+ /**
80	+ * The list of all site IDs that were queried for.
81	+ *
82	+ * @var array
83	+ */
84	+ private $sitesId;
85	+
86	+ /**
87	+ * The list of all periods that were queried for. Each period is associated with
88	+ * the period's range string. Eg,
89	+ *
90	+ * array(
91	+ * '2012-01-01,2012-01-31' => new Period(...),
92	+ * '2012-02-01,2012-02-28' => new Period(...),
93	+ * )
94	+ *
95	+ * @var \Piwik\Period[]
96	+ */
97	+ private $periods;
98	+
99	+ /**
100	+ * Constructor.
101	+ *
102	+ * @param array $dataNames @see $this->dataNames
103	+ * @param string $dataType @see $this->dataType
104	+ * @param array $sitesId @see $this->sitesId
105	+ * @param \Piwik\Period[] $periods @see $this->periods
106	+ * @param array $defaultRow @see $this->defaultRow
107	+ */
108	+ public function __construct($dataNames, $dataType, $sitesId, $periods, $segment, $defaultRow = null)
109	+ {
110	+ $this->dataNames = $dataNames;
111	+ $this->dataType = $dataType;
112	+
113	+ if ($defaultRow === null) {
114	+ $defaultRow = array_fill_keys($dataNames, 0);
115	+ }
116	+
117	+ $this->sitesId = $sitesId;
118	+
119	+ foreach ($periods as $period) {
120	+ $this->periods[$period->getRangeString()] = $period;
121	+ }
122	+
123	+ $this->segment = $segment;
124	+ $this->defaultRow = $defaultRow;
125	+ }
126	+
127	+ /**
128	+ * Returns a reference to the data for a specific site & period. If there is
129	+ * no data for the given site ID & period, it is set to the default row.
130	+ *
131	+ * @param int $idSite
132	+ * @param string $period eg, '2012-01-01,2012-01-31'
133	+ */
134	+ public function &get($idSite, $period)
135	+ {
136	+ if (!isset($this->data[$idSite][$period])) {
137	+ $this->data[$idSite][$period] = $this->defaultRow;
138	+ }
139	+ return $this->data[$idSite][$period];
140	+ }
141	+
142	+ /**
143	+ * Set data for a specific site & period. If there is no data for the given site ID & period,
144	+ * it is set to the default row.
145	+ *
146	+ * @param int $idSite
147	+ * @param string $period eg, '2012-01-01,2012-01-31'
148	+ * @param string $name eg 'nb_visits'
149	+ * @param string $value eg 5
150	+ */
151	+ public function set($idSite, $period, $name, $value)
152	+ {
153	+ $row = & $this->get($idSite, $period);
154	+ $row[$name] = $value;
155	+ }
156	+
157	+ /**
158	+ * Adds a new metadata to the data for specific site & period. If there is no
159	+ * data for the given site ID & period, it is set to the default row.
160	+ *
161	+ * Note: Site ID and period range string are two special types of metadata. Since
162	+ * the data stored in this class is indexed by site & period, this metadata is not
163	+ * stored in individual data rows.
164	+ *
165	+ * @param int $idSite
166	+ * @param string $period eg, '2012-01-01,2012-01-31'
167	+ * @param string $name The metadata name.
168	+ * @param mixed $value The metadata name.
169	+ */
170	+ public function addMetadata($idSite, $period, $name, $value)
171	+ {
172	+ $row = & $this->get($idSite, $period);
173	+ $row[self::METADATA_CONTAINER_ROW_KEY][$name] = $value;
174	+ }
175	+
176	+ /**
177	+ * Returns archive data as an array indexed by metadata.
178	+ *
179	+ * @param array $resultIndices An array mapping metadata names to pretty labels
180	+ * for them. Each archive data row will be indexed
181	+ * by the metadata specified here.
182	+ *
183	+ * Eg, array('site' => 'idSite', 'period' => 'Date')
184	+ * @return array
185	+ */
186	+ public function getIndexedArray($resultIndices)
187	+ {
188	+ $indexKeys = array_keys($resultIndices);
189	+
190	+ $result = $this->createOrderedIndex($indexKeys);
191	+ foreach ($this->data as $idSite => $rowsByPeriod) {
192	+ foreach ($rowsByPeriod as $period => $row) {
193	+ // FIXME: This hack works around a strange bug that occurs when getting
194	+ // archive IDs through ArchiveProcessing instances. When a table
195	+ // does not already exist, for some reason the archive ID for
196	+ // today (or from two days ago) will be added to the Archive
197	+ // instances list. The Archive instance will then select data
198	+ // for periods outside of the requested set.
199	+ // working around the bug here, but ideally, we need to figure
200	+ // out why incorrect idarchives are being selected.
201	+ if (empty($this->periods[$period])) {
202	+ continue;
203	+ }
204	+
205	+ $this->putRowInIndex($result, $indexKeys, $row, $idSite, $period);
206	+ }
207	+ }
208	+
209	+ return $result;
210	+ }
211	+
212	+ /**
213	+ * Returns archive data as a DataTable indexed by metadata. Indexed data will
214	+ * be represented by Map instances.
215	+ *
216	+ * @param array $resultIndices An array mapping metadata names to pretty labels
217	+ * for them. Each archive data row will be indexed
218	+ * by the metadata specified here.
219	+ *
220	+ * Eg, array('site' => 'idSite', 'period' => 'Date')
221	+ * @return DataTable\|DataTable\Map
222	+ */
223	+ public function getDataTable($resultIndices)
224	+ {
225	+ $dataTableFactory = new DataTableFactory(
226	+ $this->dataNames, $this->dataType, $this->sitesId, $this->periods, $this->segment, $this->defaultRow);
227	+
228	+ $index = $this->getIndexedArray($resultIndices);
229	+
230	+ return $dataTableFactory->make($index, $resultIndices);
231	+ }
232	+
233	+ /**
234	+ * See {@link DataTableFactory::makeMerged()}
235	+ *
236	+ * @param array $resultIndices
237	+ * @return DataTable\|DataTable\Map
238	+ * @throws Exception
239	+ */
240	+ public function getMergedDataTable($resultIndices)
241	+ {
242	+ $dataTableFactory = new DataTableFactory(
243	+ $this->dataNames, $this->dataType, $this->sitesId, $this->periods, $this->segment, $this->defaultRow);
244	+
245	+ $index = $this->getIndexedArray($resultIndices);
246	+
247	+ return $dataTableFactory->makeMerged($index, $resultIndices);
248	+ }
249	+
250	+ /**
251	+ * Returns archive data as a DataTable indexed by metadata. Indexed data will
252	+ * be represented by Map instances. Each DataTable will have
253	+ * its subtable IDs set.
254	+ *
255	+ * This function will only work if blob data was loaded and only one record
256	+ * was loaded (not including subtables of the record).
257	+ *
258	+ * @param array $resultIndices An array mapping metadata names to pretty labels
259	+ * for them. Each archive data row will be indexed
260	+ * by the metadata specified here.
261	+ *
262	+ * Eg, array('site' => 'idSite', 'period' => 'Date')
263	+ * @param int\|null $idSubTable The subtable to return.
264	+ * @param int\|null $depth max depth for subtables.
265	+ * @param bool $addMetadataSubTableId Whether to add the DB subtable ID as metadata
266	+ * to each datatable, or not.
267	+ * @throws Exception
268	+ * @return DataTable\|DataTable\Map
269	+ */
270	+ public function getExpandedDataTable($resultIndices, $idSubTable = null, $depth = null, $addMetadataSubTableId = false)
271	+ {
272	+ if ($this->dataType != 'blob') {
273	+ throw new Exception("DataCollection: cannot call getExpandedDataTable with "
274	+ . "{$this->dataType} data types. Only works with blob data.");
275	+ }
276	+
277	+ if (count($this->dataNames) !== 1) {
278	+ throw new Exception("DataCollection: cannot call getExpandedDataTable with "
279	+ . "more than one record.");
280	+ }
281	+
282	+ $dataTableFactory = new DataTableFactory(
283	+ $this->dataNames, 'blob', $this->sitesId, $this->periods, $this->segment, $this->defaultRow);
284	+ $dataTableFactory->expandDataTable($depth, $addMetadataSubTableId);
285	+ $dataTableFactory->useSubtable($idSubTable);
286	+
287	+ $index = $this->getIndexedArray($resultIndices);
288	+
289	+ return $dataTableFactory->make($index, $resultIndices);
290	+ }
291	+
292	+ /**
293	+ * Returns metadata for a data row.
294	+ *
295	+ * @param array $data The data row.
296	+ * @return array
297	+ */
298	+ public static function getDataRowMetadata($data)
299	+ {
300	+ if (isset($data[self::METADATA_CONTAINER_ROW_KEY])) {
301	+ return $data[self::METADATA_CONTAINER_ROW_KEY];
302	+ } else {
303	+ return array();
304	+ }
305	+ }
306	+
307	+ /**
308	+ * Removes all table metadata from a data row.
309	+ *
310	+ * @param array $data The data row.
311	+ */
312	+ public static function removeMetadataFromDataRow(&$data)
313	+ {
314	+ unset($data[self::METADATA_CONTAINER_ROW_KEY]);
315	+ }
316	+
317	+ /**
318	+ * Creates an empty index using a list of metadata names. If the 'site' and/or
319	+ * 'period' metadata names are supplied, empty rows are added for every site/period
320	+ * that was queried for.
321	+ *
322	+ * Using this function ensures consistent ordering in the indexed result.
323	+ *
324	+ * @param array $metadataNamesToIndexBy List of metadata names to index archive data by.
325	+ * @return array
326	+ */
327	+ private function createOrderedIndex($metadataNamesToIndexBy)
328	+ {
329	+ $result = array();
330	+
331	+ if (!empty($metadataNamesToIndexBy)) {
332	+ $metadataName = array_shift($metadataNamesToIndexBy);
333	+
334	+ if ($metadataName == DataTableFactory::TABLE_METADATA_SITE_INDEX) {
335	+ $indexKeyValues = array_values($this->sitesId);
336	+ } elseif ($metadataName == DataTableFactory::TABLE_METADATA_PERIOD_INDEX) {
337	+ $indexKeyValues = array_keys($this->periods);
338	+ }
339	+
340	+ if (empty($metadataNamesToIndexBy)) {
341	+ $result = array_fill_keys($indexKeyValues, array());
342	+ } else {
343	+ foreach ($indexKeyValues as $key) {
344	+ $result[$key] = $this->createOrderedIndex($metadataNamesToIndexBy);
345	+ }
346	+ }
347	+ }
348	+
349	+ return $result;
350	+ }
351	+
352	+ /**
353	+ * Puts an archive data row in an index.
354	+ */
355	+ private function putRowInIndex(&$index, $metadataNamesToIndexBy, $row, $idSite, $period)
356	+ {
357	+ $currentLevel = & $index;
358	+
359	+ foreach ($metadataNamesToIndexBy as $metadataName) {
360	+ if ($metadataName == DataTableFactory::TABLE_METADATA_SITE_INDEX) {
361	+ $key = $idSite;
362	+ } elseif ($metadataName == DataTableFactory::TABLE_METADATA_PERIOD_INDEX) {
363	+ $key = $period;
364	+ } else {
365	+ $key = $row[self::METADATA_CONTAINER_ROW_KEY][$metadataName];
366	+ }
367	+
368	+ if (!isset($currentLevel[$key])) {
369	+ $currentLevel[$key] = array();
370	+ }
371	+
372	+ $currentLevel = & $currentLevel[$key];
373	+ }
374	+
375	+ $currentLevel = $row;
376	+ }
377	+ }

app/core/Archive/DataTableFactory.php ADDED Viewed

	@@ -0,0 +1,594 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+
10	+ namespace Piwik\Archive;
11	+
12	+ use Piwik\API\Request;
13	+ use Piwik\Cache;
14	+ use Piwik\Cache\Transient;
15	+ use Piwik\CacheId;
16	+ use Piwik\DataTable;
17	+ use Piwik\DataTable\Row;
18	+ use Piwik\Period\Week;
19	+ use Piwik\Piwik;
20	+ use Piwik\Segment;
21	+ use Piwik\Site;
22	+
23	+ /**
24	+ * Creates a DataTable or Set instance based on an array
25	+ * index created by DataCollection.
26	+ *
27	+ * This class is only used by DataCollection.
28	+ */
29	+ class DataTableFactory
30	+ {
31	+ const TABLE_METADATA_SEGMENT_INDEX = 'segment';
32	+ const TABLE_METADATA_SEGMENT_PRETTY_INDEX = 'segmentPretty';
33	+
34	+ /**
35	+ * @see DataCollection::$dataNames.
36	+ */
37	+ private $dataNames;
38	+
39	+ /**
40	+ * @see DataCollection::$dataType.
41	+ */
42	+ private $dataType;
43	+
44	+ /**
45	+ * Whether to expand the DataTables that're created or not. Expanding a DataTable
46	+ * means creating DataTables using subtable blobs and correctly setting the subtable
47	+ * IDs of all DataTables.
48	+ *
49	+ * @var bool
50	+ */
51	+ private $expandDataTable = false;
52	+
53	+ /**
54	+ * Whether to add the subtable ID used in the database to the in-memory DataTables
55	+ * as metadata or not.
56	+ *
57	+ * @var bool
58	+ */
59	+ private $addMetadataSubtableId = false;
60	+
61	+ /**
62	+ * The maximum number of subtable levels to create when creating an expanded
63	+ * DataTable.
64	+ *
65	+ * @var int
66	+ */
67	+ private $maxSubtableDepth = null;
68	+
69	+ /**
70	+ * @see DataCollection::$sitesId.
71	+ */
72	+ private $sitesId;
73	+
74	+ /**
75	+ * @see DataCollection::$periods.
76	+ */
77	+ private $periods;
78	+
79	+ /**
80	+ * @var Segment
81	+ */
82	+ private $segment;
83	+
84	+ /**
85	+ * The ID of the subtable to create a DataTable for. Only relevant for blob data.
86	+ *
87	+ * @var int\|null
88	+ */
89	+ private $idSubtable = null;
90	+
91	+ /**
92	+ * @see DataCollection::$defaultRow.
93	+ */
94	+ private $defaultRow;
95	+
96	+ const TABLE_METADATA_SITE_INDEX = 'site';
97	+ const TABLE_METADATA_PERIOD_INDEX = 'period';
98	+
99	+ /**
100	+ * Constructor.
101	+ */
102	+ public function __construct($dataNames, $dataType, $sitesId, $periods, Segment $segment, $defaultRow)
103	+ {
104	+ $this->dataNames = $dataNames;
105	+ $this->dataType = $dataType;
106	+ $this->sitesId = $sitesId;
107	+
108	+ //here index period by string only
109	+ $this->periods = $periods;
110	+ $this->segment = $segment;
111	+ $this->defaultRow = $defaultRow;
112	+ }
113	+
114	+ /**
115	+ * Returns the ID of the site a table is related to based on the 'site' metadata entry,
116	+ * or null if there is none.
117	+ *
118	+ * @param DataTable $table
119	+ * @return int\|null
120	+ */
121	+ public static function getSiteIdFromMetadata(DataTable $table)
122	+ {
123	+ $site = $table->getMetadata('site');
124	+ if (empty($site)) {
125	+ return null;
126	+ } else {
127	+ return $site->getId();
128	+ }
129	+ }
130	+
131	+ /**
132	+ * Tells the factory instance to expand the DataTables that are created by
133	+ * creating subtables and setting the subtable IDs of rows w/ subtables correctly.
134	+ *
135	+ * @param null\|int $maxSubtableDepth max depth for subtables.
136	+ * @param bool $addMetadataSubtableId Whether to add the subtable ID used in the
137	+ * database to the in-memory DataTables as
138	+ * metadata or not.
139	+ */
140	+ public function expandDataTable($maxSubtableDepth = null, $addMetadataSubtableId = false)
141	+ {
142	+ $this->expandDataTable = true;
143	+ $this->maxSubtableDepth = $maxSubtableDepth;
144	+ $this->addMetadataSubtableId = $addMetadataSubtableId;
145	+ }
146	+
147	+ /**
148	+ * Tells the factory instance to create a DataTable using a blob with the
149	+ * supplied subtable ID.
150	+ *
151	+ * @param int $idSubtable An in-database subtable ID.
152	+ * @throws \Exception
153	+ */
154	+ public function useSubtable($idSubtable)
155	+ {
156	+ if (count($this->dataNames) !== 1) {
157	+ throw new \Exception("DataTableFactory: Getting subtables for multiple records in one"
158	+ . " archive query is not currently supported.");
159	+ }
160	+
161	+ $this->idSubtable = $idSubtable;
162	+ }
163	+
164	+ private function isNumericDataType()
165	+ {
166	+ return $this->dataType == 'numeric';
167	+ }
168	+
169	+ /**
170	+ * Creates a DataTable\|Set instance using an index of
171	+ * archive data.
172	+ *
173	+ * @param array $index @see DataCollection
174	+ * @param array $resultIndices an array mapping metadata names with pretty metadata
175	+ * labels.
176	+ * @return DataTable\|DataTable\Map
177	+ */
178	+ public function make($index, $resultIndices)
179	+ {
180	+ $keyMetadata = $this->getDefaultMetadata();
181	+
182	+ if (empty($resultIndices)) {
183	+ // for numeric data, if there's no index (and thus only 1 site & period in the query),
184	+ // we want to display every queried metric name
185	+ if (empty($index)
186	+ && $this->isNumericDataType()
187	+ ) {
188	+ $index = $this->defaultRow;
189	+ }
190	+
191	+ $dataTable = $this->createDataTable($index, $keyMetadata);
192	+ } else {
193	+ $dataTable = $this->createDataTableMapFromIndex($index, $resultIndices, $keyMetadata);
194	+ }
195	+
196	+ return $dataTable;
197	+ }
198	+
199	+ /**
200	+ * Creates a merged DataTable\|Map instance using an index of archive data similar to {@link make()}.
201	+ *
202	+ * Whereas {@link make()} creates a Map for each result index (period and\|or site), this will only create a Map
203	+ * for a period result index and move all site related indices into one dataTable. This is the same as doing
204	+ * `$dataTableFactory->make()->mergeChildren()` just much faster. It is mainly useful for reports across many sites
205	+ * eg `MultiSites.getAll`. Was done as part of https://github.com/piwik/piwik/issues/6809
206	+ *
207	+ * @param array $index @see DataCollection
208	+ * @param array $resultIndices an array mapping metadata names with pretty metadata labels.
209	+ *
210	+ * @return DataTable\|DataTable\Map
211	+ * @throws \Exception
212	+ */
213	+ public function makeMerged($index, $resultIndices)
214	+ {
215	+ if (!$this->isNumericDataType()) {
216	+ throw new \Exception('This method is supposed to work with non-numeric data types but it is not tested. To use it, remove this exception and write tests to be sure it works.');
217	+ }
218	+
219	+ $hasSiteIndex = isset($resultIndices[self::TABLE_METADATA_SITE_INDEX]);
220	+ $hasPeriodIndex = isset($resultIndices[self::TABLE_METADATA_PERIOD_INDEX]);
221	+
222	+ $isNumeric = $this->isNumericDataType();
223	+ // to be backwards compatible use a Simple table if needed as it will be formatted differently
224	+ $useSimpleDataTable = !$hasSiteIndex && $isNumeric;
225	+
226	+ if (!$hasSiteIndex) {
227	+ $firstIdSite = reset($this->sitesId);
228	+ $index = array($firstIdSite => $index);
229	+ }
230	+
231	+ if ($hasPeriodIndex) {
232	+ $dataTable = $this->makeMergedTableWithPeriodAndSiteIndex($index, $resultIndices, $useSimpleDataTable, $isNumeric);
233	+ } else {
234	+ $dataTable = $this->makeMergedWithSiteIndex($index, $useSimpleDataTable, $isNumeric);
235	+ }
236	+
237	+ return $dataTable;
238	+ }
239	+
240	+ /**
241	+ * Creates a DataTable\|Set instance using an array
242	+ * of blobs.
243	+ *
244	+ * If only one record is being queried, a single DataTable will
245	+ * be returned. Otherwise, a DataTable\Map is returned that indexes
246	+ * DataTables by record name.
247	+ *
248	+ * If expandDataTable was called, and only one record is being queried,
249	+ * the created DataTable's subtables will be expanded.
250	+ *
251	+ * @param array $blobRow
252	+ * @return DataTable\|DataTable\Map
253	+ */
254	+ private function makeFromBlobRow($blobRow, $keyMetadata)
255	+ {
256	+ if ($blobRow === false) {
257	+ $table = new DataTable();
258	+ $table->setAllTableMetadata($keyMetadata);
259	+ $this->setPrettySegmentMetadata($table);
260	+ return $table;
261	+ }
262	+
263	+ if (count($this->dataNames) === 1) {
264	+ return $this->makeDataTableFromSingleBlob($blobRow, $keyMetadata);
265	+ } else {
266	+ return $this->makeIndexedByRecordNameDataTable($blobRow, $keyMetadata);
267	+ }
268	+ }
269	+
270	+ /**
271	+ * Creates a DataTable for one record from an archive data row.
272	+ *
273	+ * @see makeFromBlobRow
274	+ *
275	+ * @param array $blobRow
276	+ * @return DataTable
277	+ */
278	+ private function makeDataTableFromSingleBlob($blobRow, $keyMetadata)
279	+ {
280	+ $recordName = reset($this->dataNames);
281	+ if ($this->idSubtable !== null) {
282	+ $recordName .= '_' . $this->idSubtable;
283	+ }
284	+
285	+ if (!empty($blobRow[$recordName])) {
286	+ $table = DataTable::fromSerializedArray($blobRow[$recordName]);
287	+ } else {
288	+ $table = new DataTable();
289	+ }
290	+
291	+ // set table metadata
292	+ $table->setAllTableMetadata(array_merge($table->getAllTableMetadata(), DataCollection::getDataRowMetadata($blobRow), $keyMetadata));
293	+ $this->setPrettySegmentMetadata($table);
294	+
295	+ if ($this->expandDataTable) {
296	+ $table->enableRecursiveFilters();
297	+ $this->setSubtables($table, $blobRow);
298	+ }
299	+
300	+ return $table;
301	+ }
302	+
303	+ /**
304	+ * Creates a DataTable for every record in an archive data row and puts them
305	+ * in a DataTable\Map instance.
306	+ *
307	+ * @param array $blobRow
308	+ * @return DataTable\Map
309	+ */
310	+ private function makeIndexedByRecordNameDataTable($blobRow, $keyMetadata)
311	+ {
312	+ $table = new DataTable\Map();
313	+ $table->setKeyName('recordName');
314	+
315	+ $tableMetadata = array_merge(DataCollection::getDataRowMetadata($blobRow), $keyMetadata);
316	+
317	+ foreach ($blobRow as $name => $blob) {
318	+ $newTable = DataTable::fromSerializedArray($blob);
319	+ $newTable->setAllTableMetadata(array_merge($newTable->getAllTableMetadata(), $tableMetadata));
320	+ $this->setPrettySegmentMetadata($newTable);
321	+
322	+ $table->addTable($newTable, $name);
323	+ }
324	+
325	+ return $table;
326	+ }
327	+
328	+ /**
329	+ * Creates a Set from an array index.
330	+ *
331	+ * @param array $index @see DataCollection
332	+ * @param array $resultIndices @see make
333	+ * @param array $keyMetadata The metadata to add to the table when it's created.
334	+ * @return DataTable\Map
335	+ */
336	+ private function createDataTableMapFromIndex($index, $resultIndices, $keyMetadata)
337	+ {
338	+ $result = new DataTable\Map();
339	+ $result->setKeyName(reset($resultIndices));
340	+ $resultIndex = key($resultIndices);
341	+
342	+ array_shift($resultIndices);
343	+
344	+ $hasIndices = !empty($resultIndices);
345	+
346	+ foreach ($index as $label => $value) {
347	+ $keyMetadata[$resultIndex] = $this->createTableIndexMetadata($resultIndex, $label);
348	+
349	+ if ($hasIndices) {
350	+ $newTable = $this->createDataTableMapFromIndex($value, $resultIndices, $keyMetadata);
351	+ } else {
352	+ $newTable = $this->createDataTable($value, $keyMetadata);
353	+ }
354	+
355	+ $result->addTable($newTable, $this->prettifyIndexLabel($resultIndex, $label));
356	+ }
357	+
358	+ return $result;
359	+ }
360	+
361	+ private function createTableIndexMetadata($resultIndex, $label)
362	+ {
363	+ if ($resultIndex === DataTableFactory::TABLE_METADATA_SITE_INDEX) {
364	+ return new Site($label);
365	+ } elseif ($resultIndex === DataTableFactory::TABLE_METADATA_PERIOD_INDEX) {
366	+ return $this->periods[$label];
367	+ }
368	+ }
369	+
370	+ /**
371	+ * Creates a DataTable instance from an index row.
372	+ *
373	+ * @param array $data An archive data row.
374	+ * @param array $keyMetadata The metadata to add to the table(s) when created.
375	+ * @return DataTable\|DataTable\Map
376	+ */
377	+ private function createDataTable($data, $keyMetadata)
378	+ {
379	+ if ($this->dataType == 'blob') {
380	+ $result = $this->makeFromBlobRow($data, $keyMetadata);
381	+ } else {
382	+ $result = $this->makeFromMetricsArray($data, $keyMetadata);
383	+ }
384	+
385	+ return $result;
386	+ }
387	+
388	+ /**
389	+ * Creates DataTables from $dataTable's subtable blobs (stored in $blobRow) and sets
390	+ * the subtable IDs of each DataTable row.
391	+ *
392	+ * @param DataTable $dataTable
393	+ * @param array $blobRow An array associating record names (w/ subtable if applicable)
394	+ * with blob values. This should hold every subtable blob for
395	+ * the loaded DataTable.
396	+ * @param int $treeLevel
397	+ */
398	+ private function setSubtables($dataTable, $blobRow, $treeLevel = 0)
399	+ {
400	+ if ($this->maxSubtableDepth
401	+ && $treeLevel >= $this->maxSubtableDepth
402	+ ) {
403	+ // unset the subtables so DataTableManager doesn't throw
404	+ foreach ($dataTable->getRowsWithoutSummaryRow() as $row) {
405	+ $row->removeSubtable();
406	+ }
407	+
408	+ return;
409	+ }
410	+
411	+ $dataName = reset($this->dataNames);
412	+
413	+ foreach ($dataTable->getRowsWithoutSummaryRow() as $row) {
414	+ $sid = $row->getIdSubDataTable();
415	+ if ($sid === null) {
416	+ continue;
417	+ }
418	+
419	+ $blobName = $dataName . "_" . $sid;
420	+ if (!empty($blobRow[$blobName])) {
421	+ $subtable = DataTable::fromSerializedArray($blobRow[$blobName]);
422	+ $subtable->setMetadata(self::TABLE_METADATA_PERIOD_INDEX, $dataTable->getMetadata(self::TABLE_METADATA_PERIOD_INDEX));
423	+ $subtable->setMetadata(self::TABLE_METADATA_SITE_INDEX, $dataTable->getMetadata(self::TABLE_METADATA_SITE_INDEX));
424	+ $subtable->setMetadata(self::TABLE_METADATA_SEGMENT_INDEX, $dataTable->getMetadata(self::TABLE_METADATA_SEGMENT_INDEX));
425	+ $subtable->setMetadata(self::TABLE_METADATA_SEGMENT_PRETTY_INDEX, $dataTable->getMetadata(self::TABLE_METADATA_SEGMENT_PRETTY_INDEX));
426	+
427	+ $this->setSubtables($subtable, $blobRow, $treeLevel + 1);
428	+
429	+ // we edit the subtable ID so that it matches the newly table created in memory
430	+ // NB: we don't overwrite the datatableid in the case we are displaying the table expanded.
431	+ if ($this->addMetadataSubtableId) {
432	+ // this will be written back to the column 'idsubdatatable' just before rendering,
433	+ // see Renderer/Php.php
434	+ $row->addMetadata('idsubdatatable_in_db', $row->getIdSubDataTable());
435	+ }
436	+
437	+ $row->setSubtable($subtable);
438	+ }
439	+ }
440	+ }
441	+
442	+ private function getDefaultMetadata()
443	+ {
444	+ return array(
445	+ DataTableFactory::TABLE_METADATA_SITE_INDEX => new Site(reset($this->sitesId)),
446	+ DataTableFactory::TABLE_METADATA_PERIOD_INDEX => reset($this->periods),
447	+ DataTableFactory::TABLE_METADATA_SEGMENT_INDEX => $this->segment->getString(),
448	+ DataTableFactory::TABLE_METADATA_SEGMENT_PRETTY_INDEX => $this->segment->getString(),
449	+ );
450	+ }
451	+
452	+ /**
453	+ * Returns the pretty version of an index label.
454	+ *
455	+ * @param string $labelType eg, 'site', 'period', etc.
456	+ * @param string $label eg, '0', '1', '2012-01-01,2012-01-31', etc.
457	+ * @return string
458	+ */
459	+ private function prettifyIndexLabel($labelType, $label)
460	+ {
461	+ if ($labelType == self::TABLE_METADATA_PERIOD_INDEX) { // prettify period labels
462	+ $period = $this->periods[$label];
463	+ $label = $period->getLabel();
464	+ if ($label === 'week' \|\| $label === 'range') {
465	+ return $period->getRangeString();
466	+ }
467	+
468	+ return $period->getPrettyString();
469	+ }
470	+ return $label;
471	+ }
472	+
473	+ /**
474	+ * @param $data
475	+ * @return DataTable\Simple
476	+ */
477	+ private function makeFromMetricsArray($data, $keyMetadata)
478	+ {
479	+ $table = new DataTable\Simple();
480	+
481	+ if (!empty($data)) {
482	+ $table->setAllTableMetadata(array_merge($table->getAllTableMetadata(), DataCollection::getDataRowMetadata($data), $keyMetadata));
483	+ $this->setPrettySegmentMetadata($table);
484	+
485	+ DataCollection::removeMetadataFromDataRow($data);
486	+
487	+ $table->addRow(new Row(array(Row::COLUMNS => $data)));
488	+ } else {
489	+ // if we're querying numeric data, we couldn't find any, and we're only
490	+ // looking for one metric, add a row w/ one column w/ value 0. this is to
491	+ // ensure that the PHP renderer outputs 0 when only one column is queried.
492	+ // w/o this code, an empty array would be created, and other parts of Piwik
493	+ // would break.
494	+ if (count($this->dataNames) == 1
495	+ && $this->isNumericDataType()
496	+ ) {
497	+ $name = reset($this->dataNames);
498	+ $table->addRow(new Row(array(Row::COLUMNS => array($name => 0))));
499	+ }
500	+
501	+ $table->setAllTableMetadata(array_merge($table->getAllTableMetadata(), $keyMetadata));
502	+ $this->setPrettySegmentMetadata($table);
503	+ }
504	+
505	+ $result = $table;
506	+ return $result;
507	+ }
508	+
509	+ private function makeMergedTableWithPeriodAndSiteIndex($index, $resultIndices, $useSimpleDataTable, $isNumeric)
510	+ {
511	+ $map = new DataTable\Map();
512	+ $map->setKeyName($resultIndices[self::TABLE_METADATA_PERIOD_INDEX]);
513	+
514	+ // we save all tables of the map in this array to be able to add rows fast
515	+ $tables = array();
516	+
517	+ foreach ($this->periods as $range => $period) {
518	+ // as the resulting table is "merged", we do only set Period metedata and no metadata for site. Instead each
519	+ // row will have an idsite metadata entry.
520	+ $metadata = array(self::TABLE_METADATA_PERIOD_INDEX => $period);
521	+
522	+ if ($useSimpleDataTable) {
523	+ $table = new DataTable\Simple();
524	+ } else {
525	+ $table = new DataTable();
526	+ }
527	+
528	+ $table->setAllTableMetadata(array_merge($table->getAllTableMetadata(), $metadata));
529	+ $this->setPrettySegmentMetadata($table);
530	+ $map->addTable($table, $this->prettifyIndexLabel(self::TABLE_METADATA_PERIOD_INDEX, $range));
531	+
532	+ $tables[$range] = $table;
533	+ }
534	+
535	+ foreach ($index as $idsite => $table) {
536	+ $rowMeta = array('idsite' => $idsite);
537	+
538	+ foreach ($table as $range => $row) {
539	+ if (!empty($row)) {
540	+ $tables[$range]->addRow(new Row(array(
541	+ Row::COLUMNS => $row,
542	+ Row::METADATA => $rowMeta)
543	+ ));
544	+ } elseif ($isNumeric) {
545	+ $tables[$range]->addRow(new Row(array(
546	+ Row::COLUMNS => $this->defaultRow,
547	+ Row::METADATA => $rowMeta)
548	+ ));
549	+ }
550	+ }
551	+ }
552	+
553	+ return $map;
554	+ }
555	+
556	+ private function makeMergedWithSiteIndex($index, $useSimpleDataTable, $isNumeric)
557	+ {
558	+ if ($useSimpleDataTable) {
559	+ $table = new DataTable\Simple();
560	+ } else {
561	+ $table = new DataTable();
562	+ }
563	+
564	+ $table->setAllTableMetadata(array(DataTableFactory::TABLE_METADATA_PERIOD_INDEX => reset($this->periods)));
565	+ $this->setPrettySegmentMetadata($table);
566	+
567	+ foreach ($index as $idsite => $row) {
568	+ if (!empty($row)) {
569	+ $table->addRow(new Row(array(
570	+ Row::COLUMNS => $row,
571	+ Row::METADATA => array('idsite' => $idsite))
572	+ ));
573	+ } elseif ($isNumeric) {
574	+ $table->addRow(new Row(array(
575	+ Row::COLUMNS => $this->defaultRow,
576	+ Row::METADATA => array('idsite' => $idsite))
577	+ ));
578	+ }
579	+ }
580	+
581	+ return $table;
582	+ }
583	+
584	+ private function setPrettySegmentMetadata(DataTable $table)
585	+ {
586	+ $site = $table->getMetadata(self::TABLE_METADATA_SITE_INDEX);
587	+ $idSite = $site ? $site->getId() : false;
588	+
589	+ $segmentPretty = $this->segment->getStoredSegmentName($idSite);
590	+
591	+ $table->setMetadata('segment', $this->segment->getString());
592	+ $table->setMetadata('segmentPretty', $segmentPretty);
593	+ }
594	+ }

app/core/Archive/Parameters.php ADDED Viewed

	@@ -0,0 +1,59 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+
10	+ namespace Piwik\Archive;
11	+
12	+ use Piwik\Period;
13	+ use Piwik\Segment;
14	+
15	+ class Parameters
16	+ {
17	+ /**
18	+ * The list of site IDs to query archive data for.
19	+ *
20	+ * @var array
21	+ */
22	+ private $idSites = array();
23	+
24	+ /**
25	+ * The list of Period's to query archive data for.
26	+ *
27	+ * @var Period[]
28	+ */
29	+ private $periods = array();
30	+
31	+ /**
32	+ * Segment applied to the visits set.
33	+ *
34	+ * @var Segment
35	+ */
36	+ private $segment;
37	+
38	+ public function getSegment()
39	+ {
40	+ return $this->segment;
41	+ }
42	+
43	+ public function __construct($idSites, $periods, Segment $segment)
44	+ {
45	+ $this->idSites = $idSites;
46	+ $this->periods = $periods;
47	+ $this->segment = $segment;
48	+ }
49	+
50	+ public function getPeriods()
51	+ {
52	+ return $this->periods;
53	+ }
54	+
55	+ public function getIdSites()
56	+ {
57	+ return $this->idSites;
58	+ }
59	+ }

app/core/ArchiveProcessor.php ADDED Viewed

	@@ -0,0 +1,640 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik;
10	+
11	+ use Exception;
12	+ use Piwik\Archive\DataTableFactory;
13	+ use Piwik\ArchiveProcessor\Parameters;
14	+ use Piwik\ArchiveProcessor\Rules;
15	+ use Piwik\DataAccess\ArchiveWriter;
16	+ use Piwik\DataAccess\LogAggregator;
17	+ use Piwik\DataTable\Manager;
18	+ use Piwik\DataTable\Map;
19	+ use Piwik\DataTable\Row;
20	+ use Piwik\Segment\SegmentExpression;
21	+
22	+ /**
23	+ * Used by {@link Piwik\Plugin\Archiver} instances to insert and aggregate archive data.
24	+ *
25	+ * ### See also
26	+ *
27	+ * - {@link Piwik\Plugin\Archiver} - to learn how plugins should implement their own analytics
28	+ * aggregation logic.
29	+ * - {@link Piwik\DataAccess\LogAggregator} - to learn how plugins can perform data aggregation
30	+ * across Piwik's log tables.
31	+ *
32	+ * ### Examples
33	+ *
34	+ * Inserting numeric data
35	+ *
36	+ * // function in an Archiver descendant
37	+ * public function aggregateDayReport()
38	+ * {
39	+ * $archiveProcessor = $this->getProcessor();
40	+ *
41	+ * $myFancyMetric = // ... calculate the metric value ...
42	+ * $archiveProcessor->insertNumericRecord('MyPlugin_myFancyMetric', $myFancyMetric);
43	+ * }
44	+ *
45	+ * Inserting serialized DataTables
46	+ *
47	+ * // function in an Archiver descendant
48	+ * public function aggregateDayReport()
49	+ * {
50	+ * $archiveProcessor = $this->getProcessor();
51	+ *
52	+ * $maxRowsInTable = Config::getInstance()->General['datatable_archiving_maximum_rows_standard'];j
53	+ *
54	+ * $dataTable = // ... build by aggregating visits ...
55	+ * $serializedData = $dataTable->getSerialized($maxRowsInTable, $maxRowsInSubtable = $maxRowsInTable,
56	+ * $columnToSortBy = Metrics::INDEX_NB_VISITS);
57	+ *
58	+ * $archiveProcessor->insertBlobRecords('MyPlugin_myFancyReport', $serializedData);
59	+ * }
60	+ *
61	+ * Aggregating archive data
62	+ *
63	+ * // function in Archiver descendant
64	+ * public function aggregateMultipleReports()
65	+ * {
66	+ * $archiveProcessor = $this->getProcessor();
67	+ *
68	+ * // aggregate a metric
69	+ * $archiveProcessor->aggregateNumericMetrics('MyPlugin_myFancyMetric');
70	+ * $archiveProcessor->aggregateNumericMetrics('MyPlugin_mySuperFancyMetric', 'max');
71	+ *
72	+ * // aggregate a report
73	+ * $archiveProcessor->aggregateDataTableRecords('MyPlugin_myFancyReport');
74	+ * }
75	+ *
76	+ */
77	+ class ArchiveProcessor
78	+ {
79	+ /**
80	+ * @var \Piwik\DataAccess\ArchiveWriter
81	+ */
82	+ private $archiveWriter;
83	+
84	+ /**
85	+ * @var \Piwik\DataAccess\LogAggregator
86	+ */
87	+ private $logAggregator;
88	+
89	+ /**
90	+ * @var Archive
91	+ */
92	+ public $archive = null;
93	+
94	+ /**
95	+ * @var Parameters
96	+ */
97	+ private $params;
98	+
99	+ /**
100	+ * @var int
101	+ */
102	+ private $numberOfVisits = false;
103	+
104	+ private $numberOfVisitsConverted = false;
105	+
106	+ /**
107	+ * If true, unique visitors are not calculated when we are aggregating data for multiple sites.
108	+ * The `[General] enable_processing_unique_visitors_multiple_sites` INI config option controls
109	+ * the value of this variable.
110	+ *
111	+ * @var bool
112	+ */
113	+ private $skipUniqueVisitorsCalculationForMultipleSites = true;
114	+
115	+ public function __construct(Parameters $params, ArchiveWriter $archiveWriter, LogAggregator $logAggregator)
116	+ {
117	+ $this->params = $params;
118	+ $this->logAggregator = $logAggregator;
119	+ $this->archiveWriter = $archiveWriter;
120	+
121	+ $this->skipUniqueVisitorsCalculationForMultipleSites = Rules::shouldSkipUniqueVisitorsCalculationForMultipleSites();
122	+ }
123	+
124	+ protected function getArchive()
125	+ {
126	+ if (empty($this->archive)) {
127	+ $subPeriods = $this->params->getSubPeriods();
128	+ $idSites = $this->params->getIdSites();
129	+ $this->archive = Archive::factory($this->params->getSegment(), $subPeriods, $idSites);
130	+ }
131	+
132	+ return $this->archive;
133	+ }
134	+
135	+ public function setNumberOfVisits($visits, $visitsConverted)
136	+ {
137	+ $this->numberOfVisits = $visits;
138	+ $this->numberOfVisitsConverted = $visitsConverted;
139	+ }
140	+
141	+ /**
142	+ * Returns the {@link Parameters} object containing the site, period and segment we're archiving
143	+ * data for.
144	+ *
145	+ * @return Parameters
146	+ * @api
147	+ */
148	+ public function getParams()
149	+ {
150	+ return $this->params;
151	+ }
152	+
153	+ /**
154	+ * Returns a `{@link Piwik\DataAccess\LogAggregator}` instance for the site, period and segment this
155	+ * ArchiveProcessor will insert archive data for.
156	+ *
157	+ * @return LogAggregator
158	+ * @api
159	+ */
160	+ public function getLogAggregator()
161	+ {
162	+ return $this->logAggregator;
163	+ }
164	+
165	+ /**
166	+ * Array of (column name before => column name renamed) of the columns for which sum operation is invalid.
167	+ * These columns will be renamed as per this mapping.
168	+ * @var array
169	+ */
170	+ protected static $columnsToRenameAfterAggregation = array(
171	+ Metrics::INDEX_NB_UNIQ_VISITORS => Metrics::INDEX_SUM_DAILY_NB_UNIQ_VISITORS,
172	+ Metrics::INDEX_NB_USERS => Metrics::INDEX_SUM_DAILY_NB_USERS,
173	+ );
174	+
175	+ /**
176	+ * Sums records for every subperiod of the current period and inserts the result as the record
177	+ * for this period.
178	+ *
179	+ * DataTables are summed recursively so subtables will be summed as well.
180	+ *
181	+ * @param string\|array $recordNames Name(s) of the report we are aggregating, eg, `'Referrers_type'`.
182	+ * @param int $maximumRowsInDataTableLevelZero Maximum number of rows allowed in the top level DataTable.
183	+ * @param int $maximumRowsInSubDataTable Maximum number of rows allowed in each subtable.
184	+ * @param string $columnToSortByBeforeTruncation The name of the column to sort by before truncating a DataTable.
185	+ * @param array $columnsAggregationOperation Operations for aggregating columns, see {@link Row::sumRow()}.
186	+ * @param array $columnsToRenameAfterAggregation Columns mapped to new names for columns that must change names
187	+ * when summed because they cannot be summed, eg,
188	+ * `array('nb_uniq_visitors' => 'sum_daily_nb_uniq_visitors')`.
189	+ * @param bool\|array $countRowsRecursive if set to true, will calculate the recursive rows count for all record names
190	+ * which makes it slower. If you only need it for some records pass an array of
191	+ * recordNames that defines for which ones you need a recursive row count.
192	+ * @return array Returns the row counts of each aggregated report before truncation, eg,
193	+ *
194	+ * array(
195	+ * 'report1' => array('level0' => $report1->getRowsCount,
196	+ * 'recursive' => $report1->getRowsCountRecursive()),
197	+ * 'report2' => array('level0' => $report2->getRowsCount,
198	+ * 'recursive' => $report2->getRowsCountRecursive()),
199	+ * ...
200	+ * )
201	+ * @api
202	+ */
203	+ public function aggregateDataTableRecords($recordNames,
204	+ $maximumRowsInDataTableLevelZero = null,
205	+ $maximumRowsInSubDataTable = null,
206	+ $columnToSortByBeforeTruncation = null,
207	+ &$columnsAggregationOperation = null,
208	+ $columnsToRenameAfterAggregation = null,
209	+ $countRowsRecursive = true)
210	+ {
211	+ if (!is_array($recordNames)) {
212	+ $recordNames = array($recordNames);
213	+ }
214	+
215	+ $nameToCount = array();
216	+ foreach ($recordNames as $recordName) {
217	+ $latestUsedTableId = Manager::getInstance()->getMostRecentTableId();
218	+
219	+ $table = $this->aggregateDataTableRecord($recordName, $columnsAggregationOperation, $columnsToRenameAfterAggregation);
220	+
221	+ $nameToCount[$recordName]['level0'] = $table->getRowsCount();
222	+ if ($countRowsRecursive === true \|\| (is_array($countRowsRecursive) && in_array($recordName, $countRowsRecursive))) {
223	+ $nameToCount[$recordName]['recursive'] = $table->getRowsCountRecursive();
224	+ }
225	+
226	+ $blob = $table->getSerialized($maximumRowsInDataTableLevelZero, $maximumRowsInSubDataTable, $columnToSortByBeforeTruncation);
227	+ Common::destroy($table);
228	+ $this->insertBlobRecord($recordName, $blob);
229	+
230	+ unset($blob);
231	+ DataTable\Manager::getInstance()->deleteAll($latestUsedTableId);
232	+ }
233	+
234	+ return $nameToCount;
235	+ }
236	+
237	+ /**
238	+ * Aggregates one or more metrics for every subperiod of the current period and inserts the results
239	+ * as metrics for the current period.
240	+ *
241	+ * @param array\|string $columns Array of metric names to aggregate.
242	+ * @param bool\|string $operationToApply The operation to apply to the metric. Either `'sum'`, `'max'` or `'min'`.
243	+ * @return array\|int Returns the array of aggregate values. If only one metric was aggregated,
244	+ * the aggregate value will be returned as is, not in an array.
245	+ * For example, if `array('nb_visits', 'nb_hits')` is supplied for `$columns`,
246	+ *
247	+ * array(
248	+ * 'nb_visits' => 3040,
249	+ * 'nb_hits' => 405
250	+ * )
251	+ *
252	+ * could be returned. If `array('nb_visits')` or `'nb_visits'` is used for `$columns`,
253	+ * then `3040` would be returned.
254	+ * @api
255	+ */
256	+ public function aggregateNumericMetrics($columns, $operationToApply = false)
257	+ {
258	+ $metrics = $this->getAggregatedNumericMetrics($columns, $operationToApply);
259	+
260	+ foreach ($metrics as $column => $value) {
261	+ $value = Common::forceDotAsSeparatorForDecimalPoint($value);
262	+ $this->archiveWriter->insertRecord($column, $value);
263	+ }
264	+ // if asked for only one field to sum
265	+ if (count($metrics) == 1) {
266	+ return reset($metrics);
267	+ }
268	+
269	+ // returns the array of records once summed
270	+ return $metrics;
271	+ }
272	+
273	+ public function getNumberOfVisits()
274	+ {
275	+ if ($this->numberOfVisits === false) {
276	+ throw new Exception("visits should have been set here");
277	+ }
278	+ return $this->numberOfVisits;
279	+ }
280	+
281	+ public function getNumberOfVisitsConverted()
282	+ {
283	+ return $this->numberOfVisitsConverted;
284	+ }
285	+
286	+ /**
287	+ * Caches multiple numeric records in the archive for this processor's site, period
288	+ * and segment.
289	+ *
290	+ * @param array $numericRecords A name-value mapping of numeric values that should be
291	+ * archived, eg,
292	+ *
293	+ * array('Referrers_distinctKeywords' => 23, 'Referrers_distinctCampaigns' => 234)
294	+ * @api
295	+ */
296	+ public function insertNumericRecords($numericRecords)
297	+ {
298	+ foreach ($numericRecords as $name => $value) {
299	+ $this->insertNumericRecord($name, $value);
300	+ }
301	+ }
302	+
303	+ /**
304	+ * Caches a single numeric record in the archive for this processor's site, period and
305	+ * segment.
306	+ *
307	+ * Numeric values are not inserted if they equal `0`.
308	+ *
309	+ * @param string $name The name of the numeric value, eg, `'Referrers_distinctKeywords'`.
310	+ * @param float $value The numeric value.
311	+ * @api
312	+ */
313	+ public function insertNumericRecord($name, $value)
314	+ {
315	+ $value = round($value, 2);
316	+ $value = Common::forceDotAsSeparatorForDecimalPoint($value);
317	+
318	+ $this->archiveWriter->insertRecord($name, $value);
319	+ }
320	+
321	+ /**
322	+ * Caches one or more blob records in the archive for this processor's site, period
323	+ * and segment.
324	+ *
325	+ * @param string $name The name of the record, eg, 'Referrers_type'.
326	+ * @param string\|array $values A blob string or an array of blob strings. If an array
327	+ * is used, the first element in the array will be inserted
328	+ * with the `$name` name. The others will be inserted with
329	+ * `$name . '_' . $index` as the record name (where $index is
330	+ * the index of the blob record in `$values`).
331	+ * @api
332	+ */
333	+ public function insertBlobRecord($name, $values)
334	+ {
335	+ $this->archiveWriter->insertBlobRecord($name, $values);
336	+ }
337	+
338	+ /**
339	+ * This method selects all DataTables that have the name $name over the period.
340	+ * All these DataTables are then added together, and the resulting DataTable is returned.
341	+ *
342	+ * @param string $name
343	+ * @param array $columnsAggregationOperation Operations for aggregating columns, @see Row::sumRow()
344	+ * @param array $columnsToRenameAfterAggregation columns in the array (old name, new name) to be renamed as the sum operation is not valid on them (eg. nb_uniq_visitors->sum_daily_nb_uniq_visitors)
345	+ * @return DataTable
346	+ */
347	+ protected function aggregateDataTableRecord($name, $columnsAggregationOperation = null, $columnsToRenameAfterAggregation = null)
348	+ {
349	+ try {
350	+ ErrorHandler::pushFatalErrorBreadcrumb(__CLASS__, ['name' => $name]);
351	+
352	+ // By default we shall aggregate all sub-tables.
353	+ $dataTable = $this->getArchive()->getDataTableExpanded($name, $idSubTable = null, $depth = null, $addMetadataSubtableId = false);
354	+
355	+ $columnsRenamed = false;
356	+
357	+ if ($dataTable instanceof Map) {
358	+ $columnsRenamed = true;
359	+ // see https://github.com/piwik/piwik/issues/4377
360	+ $self = $this;
361	+ $dataTable->filter(function ($table) use ($self, $columnsToRenameAfterAggregation) {
362	+
363	+ if ($self->areColumnsNotAlreadyRenamed($table)) {
364	+ /**
365	+ * This makes archiving and range dates a lot faster. Imagine we archive a week, then we will
366	+ * rename all columns of each 7 day archives. Afterwards we know the columns will be replaced in a
367	+ * week archive. When generating month archives, which uses mostly week archives, we do not have
368	+ * to replace those columns for the week archives again since we can be sure they were already
369	+ * replaced. Same when aggregating year and range archives. This can save up 10% or more when
370	+ * aggregating Month, Year and Range archives.
371	+ */
372	+ $self->renameColumnsAfterAggregation($table, $columnsToRenameAfterAggregation);
373	+ }
374	+ });
375	+ }
376	+
377	+ $dataTable = $this->getAggregatedDataTableMap($dataTable, $columnsAggregationOperation);
378	+
379	+ if (!$columnsRenamed) {
380	+ $this->renameColumnsAfterAggregation($dataTable, $columnsToRenameAfterAggregation);
381	+ }
382	+ } finally {
383	+ ErrorHandler::popFatalErrorBreadcrumb();
384	+ }
385	+
386	+ return $dataTable;
387	+ }
388	+
389	+ /**
390	+ * Note: public only for use in closure in PHP 5.3.
391	+ *
392	+ * @param $table
393	+ * @return \Piwik\Period
394	+ */
395	+ public function areColumnsNotAlreadyRenamed($table)
396	+ {
397	+ $period = $table->getMetadata(DataTableFactory::TABLE_METADATA_PERIOD_INDEX);
398	+
399	+ return !$period \|\| $period->getLabel() === 'day';
400	+ }
401	+
402	+ protected function getOperationForColumns($columns, $defaultOperation)
403	+ {
404	+ $operationForColumn = array();
405	+ foreach ($columns as $name) {
406	+ $operation = $defaultOperation;
407	+ if (empty($operation)) {
408	+ $operation = $this->guessOperationForColumn($name);
409	+ }
410	+ $operationForColumn[$name] = $operation;
411	+ }
412	+ return $operationForColumn;
413	+ }
414	+
415	+ protected function enrichWithUniqueVisitorsMetric(Row $row)
416	+ {
417	+ // skip unique visitors metrics calculation if calculating for multiple sites is disabled
418	+ if (!$this->getParams()->isSingleSite()
419	+ && $this->skipUniqueVisitorsCalculationForMultipleSites
420	+ ) {
421	+ return;
422	+ }
423	+
424	+ if ($row->getColumn('nb_uniq_visitors') === false
425	+ && $row->getColumn('nb_users') === false
426	+ ) {
427	+ return;
428	+ }
429	+
430	+ if (!SettingsPiwik::isUniqueVisitorsEnabled($this->getParams()->getPeriod()->getLabel())) {
431	+ $row->deleteColumn('nb_uniq_visitors');
432	+ $row->deleteColumn('nb_users');
433	+ return;
434	+ }
435	+
436	+ $metrics = array(
437	+ Metrics::INDEX_NB_USERS
438	+ );
439	+
440	+ if ($this->getParams()->isSingleSite()) {
441	+ $uniqueVisitorsMetric = Metrics::INDEX_NB_UNIQ_VISITORS;
442	+ } else {
443	+ if (!SettingsPiwik::isSameFingerprintAcrossWebsites()) {
444	+ throw new Exception("Processing unique visitors across websites is enabled for this instance,
445	+ but to process this metric you must first set enable_fingerprinting_across_websites=1
446	+ in the config file, under the [Tracker] section.");
447	+ }
448	+ $uniqueVisitorsMetric = Metrics::INDEX_NB_UNIQ_FINGERPRINTS;
449	+ }
450	+ $metrics[] = $uniqueVisitorsMetric;
451	+
452	+ $uniques = $this->computeNbUniques($metrics);
453	+
454	+ // see edge case as described in https://github.com/piwik/piwik/issues/9357 where uniq_visitors might be higher
455	+ // than visits because we archive / process it after nb_visits. Between archiving nb_visits and nb_uniq_visitors
456	+ // there could have been a new visit leading to a higher nb_unique_visitors than nb_visits which is not possible
457	+ // by definition. In this case we simply use the visits metric instead of unique visitors metric.
458	+ $visits = $row->getColumn('nb_visits');
459	+ if ($visits !== false && $uniques[$uniqueVisitorsMetric] !== false) {
460	+ $uniques[$uniqueVisitorsMetric] = min($uniques[$uniqueVisitorsMetric], $visits);
461	+ }
462	+
463	+ $row->setColumn('nb_uniq_visitors', $uniques[$uniqueVisitorsMetric]);
464	+ $row->setColumn('nb_users', $uniques[Metrics::INDEX_NB_USERS]);
465	+ }
466	+
467	+ protected function guessOperationForColumn($column)
468	+ {
469	+ if (strpos($column, 'max_') === 0) {
470	+ return 'max';
471	+ }
472	+ if (strpos($column, 'min_') === 0) {
473	+ return 'min';
474	+ }
475	+ return 'sum';
476	+ }
477	+
478	+ /**
479	+ * Processes number of unique visitors for the given period
480	+ *
481	+ * This is the only Period metric (ie. week/month/year/range) that we process from the logs directly,
482	+ * since unique visitors cannot be summed like other metrics.
483	+ *
484	+ * @param array Metrics Ids for which to aggregates count of values
485	+ * @return array of metrics, where the key is metricid and the value is the metric value
486	+ */
487	+ protected function computeNbUniques($metrics)
488	+ {
489	+ $logAggregator = $this->getLogAggregator();
490	+ $query = $logAggregator->queryVisitsByDimension(array(), false, array(), $metrics);
491	+ $data = $query->fetch();
492	+ return $data;
493	+ }
494	+
495	+ /**
496	+ * If the DataTable is a Map, sums all DataTable in the map and return the DataTable.
497	+ *
498	+ *
499	+ * @param $data DataTable\|DataTable\Map
500	+ * @param $columnsToRenameAfterAggregation array
501	+ * @return DataTable
502	+ */
503	+ protected function getAggregatedDataTableMap($data, $columnsAggregationOperation)
504	+ {
505	+ $table = new DataTable();
506	+
507	+ if (!empty($columnsAggregationOperation)) {
508	+ $table->setMetadata(DataTable::COLUMN_AGGREGATION_OPS_METADATA_NAME, $columnsAggregationOperation);
509	+ }
510	+
511	+ if ($data instanceof DataTable\Map) {
512	+ // as $date => $tableToSum
513	+ $this->aggregatedDataTableMapsAsOne($data, $table);
514	+ } else {
515	+ $table->addDataTable($data);
516	+ }
517	+
518	+ return $table;
519	+ }
520	+
521	+ /**
522	+ * Aggregates the DataTable\Map into the destination $aggregated
523	+ * @param $map
524	+ * @param $aggregated
525	+ */
526	+ protected function aggregatedDataTableMapsAsOne(Map $map, DataTable $aggregated)
527	+ {
528	+ foreach ($map->getDataTables() as $tableToAggregate) {
529	+ if ($tableToAggregate instanceof Map) {
530	+ $this->aggregatedDataTableMapsAsOne($tableToAggregate, $aggregated);
531	+ } else {
532	+ $aggregated->addDataTable($tableToAggregate);
533	+ }
534	+ }
535	+ }
536	+
537	+ /**
538	+ * Note: public only for use in closure in PHP 5.3.
539	+ */
540	+ public function renameColumnsAfterAggregation(DataTable $table, $columnsToRenameAfterAggregation = null)
541	+ {
542	+ // Rename columns after aggregation
543	+ if (is_null($columnsToRenameAfterAggregation)) {
544	+ $columnsToRenameAfterAggregation = self::$columnsToRenameAfterAggregation;
545	+ }
546	+
547	+ if (empty($columnsToRenameAfterAggregation)) {
548	+ return;
549	+ }
550	+
551	+ foreach ($table->getRows() as $row) {
552	+ foreach ($columnsToRenameAfterAggregation as $oldName => $newName) {
553	+ $row->renameColumn($oldName, $newName);
554	+ }
555	+
556	+ $subTable = $row->getSubtable();
557	+ if ($subTable) {
558	+ $this->renameColumnsAfterAggregation($subTable, $columnsToRenameAfterAggregation);
559	+ }
560	+ }
561	+ }
562	+
563	+ protected function getAggregatedNumericMetrics($columns, $operationToApply)
564	+ {
565	+ if (!is_array($columns)) {
566	+ $columns = array($columns);
567	+ }
568	+
569	+ $operationForColumn = $this->getOperationForColumns($columns, $operationToApply);
570	+
571	+ $dataTable = $this->getArchive()->getDataTableFromNumeric($columns);
572	+
573	+ $results = $this->getAggregatedDataTableMap($dataTable, $operationForColumn);
574	+ if ($results->getRowsCount() > 1) {
575	+ throw new Exception("A DataTable is an unexpected state:" . var_export($results, true));
576	+ }
577	+
578	+ $rowMetrics = $results->getFirstRow();
579	+ if ($rowMetrics === false) {
580	+ $rowMetrics = new Row;
581	+ }
582	+ $this->enrichWithUniqueVisitorsMetric($rowMetrics);
583	+ $this->renameColumnsAfterAggregation($results, self::$columnsToRenameAfterAggregation);
584	+
585	+ $metrics = $rowMetrics->getColumns();
586	+
587	+ foreach ($columns as $name) {
588	+ if (!isset($metrics[$name])) {
589	+ $metrics[$name] = 0;
590	+ }
591	+ }
592	+
593	+ return $metrics;
594	+ }
595	+
596	+ /**
597	+ * Initiate archiving for a plugin during an ongoing archiving. The plugin can be another
598	+ * plugin or the same plugin.
599	+ *
600	+ * This method should be called during archiving when one plugin uses the report of another
601	+ * plugin with a segment. It will ensure reports for that segment & plugin will be archived
602	+ * without initiating archiving for every plugin with that segment (which would be a performance
603	+ * killer).
604	+ *
605	+ * @param string $plugin
606	+ * @param string $segment
607	+ */
608	+ public function processDependentArchive($plugin, $segment)
609	+ {
610	+ $params = $this->getParams();
611	+ if (!$params->isRootArchiveRequest()) { // prevent all recursion
612	+ return;
613	+ }
614	+
615	+ $idSites = [$params->getSite()->getId()];
616	+
617	+ $newSegment = Segment::combine($params->getSegment()->getString(), SegmentExpression::AND_DELIMITER, $segment);
618	+ if ($newSegment === $segment && $params->getRequestedPlugin() === $plugin) { // being processed now
619	+ return;
620	+ }
621	+
622	+ $newSegment = new Segment($newSegment, $idSites);
623	+ if (ArchiveProcessor\Rules::isSegmentPreProcessed($idSites, $newSegment)) {
624	+ // will be processed anyway
625	+ return;
626	+ }
627	+
628	+ $parameters = new ArchiveProcessor\Parameters($params->getSite(), $params->getPeriod(), $newSegment);
629	+ $parameters->onlyArchiveRequestedPlugin();
630	+ $parameters->setIsRootArchiveRequest(false);
631	+
632	+ $archiveLoader = new ArchiveProcessor\Loader($parameters);
633	+ $archiveLoader->prepareArchive($plugin);
634	+ }
635	+
636	+ public function getArchiveWriter()
637	+ {
638	+ return $this->archiveWriter;
639	+ }
640	+ }

app/core/ArchiveProcessor/ArchivingStatus.php ADDED Viewed

	@@ -0,0 +1,111 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+
10	+ namespace Piwik\ArchiveProcessor;
11	+
12	+ use Piwik\Common;
13	+ use Piwik\Concurrency\Lock;
14	+ use Piwik\Concurrency\LockBackend;
15	+ use Piwik\Container\StaticContainer;
16	+ use Piwik\SettingsPiwik;
17	+
18	+ class ArchivingStatus
19	+ {
20	+ const LOCK_KEY_PREFIX = 'Archiving';
21	+ const DEFAULT_ARCHIVING_TTL = 7200; // 2 hours
22	+
23	+ /**
24	+ * @var LockBackend
25	+ */
26	+ private $lockBackend;
27	+
28	+ /**
29	+ * @var int
30	+ */
31	+ private $archivingTTLSecs;
32	+
33	+ /**
34	+ * @var Lock[]
35	+ */
36	+ private $lockStack = [];
37	+
38	+ private $pid;
39	+
40	+ public function __construct(LockBackend $lockBackend, $archivingTTLSecs = self::DEFAULT_ARCHIVING_TTL)
41	+ {
42	+ $this->lockBackend = $lockBackend;
43	+ $this->archivingTTLSecs = $archivingTTLSecs;
44	+ $this->pid = Common::getProcessId();
45	+ }
46	+
47	+ public function archiveStarted(Parameters $params)
48	+ {
49	+ $lock = $this->makeArchivingLock($params);
50	+ $lock->acquireLock($this->getInstanceProcessId(), $this->archivingTTLSecs);
51	+ array_push($this->lockStack, $lock);
52	+ }
53	+
54	+ public function archiveFinished()
55	+ {
56	+ $lock = array_pop($this->lockStack);
57	+ $lock->unlock();
58	+ }
59	+
60	+ public function getCurrentArchivingLock()
61	+ {
62	+ if (empty($this->lockStack)) {
63	+ return null;
64	+ }
65	+ return end($this->lockStack);
66	+ }
67	+
68	+ public function getSitesCurrentlyArchiving()
69	+ {
70	+ $lockMeta = new Lock($this->lockBackend, self::LOCK_KEY_PREFIX . '.');
71	+ $acquiredLocks = $lockMeta->getAllAcquiredLockKeys();
72	+
73	+ $sitesCurrentlyArchiving = [];
74	+ foreach ($acquiredLocks as $lockKey) {
75	+ $parts = explode('.', $lockKey);
76	+ if (!isset($parts[1])) {
77	+ continue;
78	+ }
79	+ $sitesCurrentlyArchiving[] = (int) $parts[1];
80	+ }
81	+ $sitesCurrentlyArchiving = array_unique($sitesCurrentlyArchiving);
82	+ $sitesCurrentlyArchiving = array_values($sitesCurrentlyArchiving);
83	+
84	+ return $sitesCurrentlyArchiving;
85	+ }
86	+
87	+ /**
88	+ * @return Lock
89	+ */
90	+ private function makeArchivingLock(Parameters $params)
91	+ {
92	+ $doneFlag = Rules::getDoneStringFlagFor([$params->getSite()->getId()], $params->getSegment(),
93	+ $params->getPeriod()->getLabel(), $params->getRequestedPlugin());
94	+
95	+ $lockKeyParts = [
96	+ self::LOCK_KEY_PREFIX,
97	+ $params->getSite()->getId(),
98	+
99	+ // md5 to keep it within the 70 char limit in the table
100	+ md5($params->getPeriod()->getId() . $params->getPeriod()->getRangeString() . $doneFlag),
101	+ ];
102	+
103	+ $lockKeyPrefix = implode('.', $lockKeyParts);
104	+ return new Lock(StaticContainer::get(LockBackend::class), $lockKeyPrefix, $this->archivingTTLSecs);
105	+ }
106	+
107	+ private function getInstanceProcessId()
108	+ {
109	+ return SettingsPiwik::getPiwikInstanceId() . '.' . $this->pid;
110	+ }
111	+ }

app/core/ArchiveProcessor/Loader.php ADDED Viewed

	@@ -0,0 +1,245 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik\ArchiveProcessor;
10	+
11	+ use Piwik\Cache;
12	+ use Piwik\Config;
13	+ use Piwik\Container\StaticContainer;
14	+ use Piwik\Context;
15	+ use Piwik\DataAccess\ArchiveSelector;
16	+ use Piwik\Date;
17	+ use Piwik\Piwik;
18	+
19	+ /**
20	+ * This class uses PluginsArchiver class to trigger data aggregation and create archives.
21	+ */
22	+ class Loader
23	+ {
24	+ /**
25	+ * Idarchive in the DB for the requested archive
26	+ *
27	+ * @var int
28	+ */
29	+ protected $idArchive;
30	+
31	+ /**
32	+ * @var Parameters
33	+ */
34	+ protected $params;
35	+
36	+ public function __construct(Parameters $params)
37	+ {
38	+ $this->params = $params;
39	+ }
40	+
41	+ /**
42	+ * @return bool
43	+ */
44	+ protected function isThereSomeVisits($visits)
45	+ {
46	+ return $visits > 0;
47	+ }
48	+
49	+ /**
50	+ * @return bool
51	+ */
52	+ protected function mustProcessVisitCount($visits)
53	+ {
54	+ return $visits === false;
55	+ }
56	+
57	+ public function prepareArchive($pluginName)
58	+ {
59	+ return Context::changeIdSite($this->params->getSite()->getId(), function () use ($pluginName) {
60	+ return $this->prepareArchiveImpl($pluginName);
61	+ });
62	+ }
63	+
64	+ private function prepareArchiveImpl($pluginName)
65	+ {
66	+ $this->params->setRequestedPlugin($pluginName);
67	+
68	+ list($idArchive, $visits, $visitsConverted) = $this->loadExistingArchiveIdFromDb();
69	+ if (!empty($idArchive)) {
70	+ return $idArchive;
71	+ }
72	+
73	+ /** @var ArchivingStatus $archivingStatus */
74	+ $archivingStatus = StaticContainer::get(ArchivingStatus::class);
75	+ $archivingStatus->archiveStarted($this->params);
76	+
77	+ try {
78	+ list($visits, $visitsConverted) = $this->prepareCoreMetricsArchive($visits, $visitsConverted);
79	+ list($idArchive, $visits) = $this->prepareAllPluginsArchive($visits, $visitsConverted);
80	+ } finally {
81	+ $archivingStatus->archiveFinished();
82	+ }
83	+
84	+ if ($this->isThereSomeVisits($visits) \|\| PluginsArchiver::doesAnyPluginArchiveWithoutVisits()) {
85	+ return $idArchive;
86	+ }
87	+
88	+ return false;
89	+ }
90	+
91	+ /**
92	+ * Prepares the core metrics if needed.
93	+ *
94	+ * @param $visits
95	+ * @return array
96	+ */
97	+ protected function prepareCoreMetricsArchive($visits, $visitsConverted)
98	+ {
99	+ $createSeparateArchiveForCoreMetrics = $this->mustProcessVisitCount($visits)
100	+ && !$this->doesRequestedPluginIncludeVisitsSummary();
101	+
102	+ if ($createSeparateArchiveForCoreMetrics) {
103	+ $requestedPlugin = $this->params->getRequestedPlugin();
104	+
105	+ $this->params->setRequestedPlugin('VisitsSummary');
106	+
107	+ $pluginsArchiver = new PluginsArchiver($this->params);
108	+ $metrics = $pluginsArchiver->callAggregateCoreMetrics();
109	+ $pluginsArchiver->finalizeArchive();
110	+
111	+ $this->params->setRequestedPlugin($requestedPlugin);
112	+
113	+ $visits = $metrics['nb_visits'];
114	+ $visitsConverted = $metrics['nb_visits_converted'];
115	+ }
116	+
117	+ return array($visits, $visitsConverted);
118	+ }
119	+
120	+ protected function prepareAllPluginsArchive($visits, $visitsConverted)
121	+ {
122	+ $pluginsArchiver = new PluginsArchiver($this->params);
123	+
124	+ if ($this->mustProcessVisitCount($visits)
125	+ \|\| $this->doesRequestedPluginIncludeVisitsSummary()
126	+ ) {
127	+ $metrics = $pluginsArchiver->callAggregateCoreMetrics();
128	+ $visits = $metrics['nb_visits'];
129	+ $visitsConverted = $metrics['nb_visits_converted'];
130	+ }
131	+
132	+ $forceArchivingWithoutVisits = !$this->isThereSomeVisits($visits) && $this->shouldArchiveForSiteEvenWhenNoVisits();
133	+ $pluginsArchiver->callAggregateAllPlugins($visits, $visitsConverted, $forceArchivingWithoutVisits);
134	+
135	+ $idArchive = $pluginsArchiver->finalizeArchive();
136	+
137	+ return array($idArchive, $visits);
138	+ }
139	+
140	+ protected function doesRequestedPluginIncludeVisitsSummary()
141	+ {
142	+ $processAllReportsIncludingVisitsSummary =
143	+ Rules::shouldProcessReportsAllPlugins($this->params->getIdSites(), $this->params->getSegment(), $this->params->getPeriod()->getLabel());
144	+ $doesRequestedPluginIncludeVisitsSummary = $processAllReportsIncludingVisitsSummary
145	+ \|\| $this->params->getRequestedPlugin() == 'VisitsSummary';
146	+ return $doesRequestedPluginIncludeVisitsSummary;
147	+ }
148	+
149	+ protected function isArchivingForcedToTrigger()
150	+ {
151	+ $period = $this->params->getPeriod()->getLabel();
152	+ $debugSetting = 'always_archive_data_period'; // default
153	+
154	+ if ($period == 'day') {
155	+ $debugSetting = 'always_archive_data_day';
156	+ } elseif ($period == 'range') {
157	+ $debugSetting = 'always_archive_data_range';
158	+ }
159	+
160	+ return (bool) Config::getInstance()->Debug[$debugSetting];
161	+ }
162	+
163	+ /**
164	+ * Returns the idArchive if the archive is available in the database for the requested plugin.
165	+ * Returns false if the archive needs to be processed.
166	+ *
167	+ * (public for tests)
168	+ *
169	+ * @return array
170	+ */
171	+ public function loadExistingArchiveIdFromDb()
172	+ {
173	+ $noArchiveFound = array(false, false, false);
174	+
175	+ if ($this->isArchivingForcedToTrigger()) {
176	+ return $noArchiveFound;
177	+ }
178	+
179	+ $minDatetimeArchiveProcessedUTC = $this->getMinTimeArchiveProcessed();
180	+ $idAndVisits = ArchiveSelector::getArchiveIdAndVisits($this->params, $minDatetimeArchiveProcessedUTC);
181	+
182	+ if (!$idAndVisits) {
183	+ return $noArchiveFound;
184	+ }
185	+
186	+ return $idAndVisits;
187	+ }
188	+
189	+ /**
190	+ * Returns the minimum archive processed datetime to look at. Only public for tests.
191	+ *
192	+ * @return int\|bool Datetime timestamp, or false if must look at any archive available
193	+ */
194	+ protected function getMinTimeArchiveProcessed()
195	+ {
196	+ $endDateTimestamp = self::determineIfArchivePermanent($this->params->getDateEnd());
197	+ if ($endDateTimestamp) {
198	+ // past archive
199	+ return $endDateTimestamp;
200	+ }
201	+ $dateStart = $this->params->getDateStart();
202	+ $period = $this->params->getPeriod();
203	+ $segment = $this->params->getSegment();
204	+ $site = $this->params->getSite();
205	+ // in-progress archive
206	+ return Rules::getMinTimeProcessedForInProgressArchive($dateStart, $period, $segment, $site);
207	+ }
208	+
209	+ protected static function determineIfArchivePermanent(Date $dateEnd)
210	+ {
211	+ $now = time();
212	+ $endTimestampUTC = strtotime($dateEnd->getDateEndUTC());
213	+
214	+ if ($endTimestampUTC <= $now) {
215	+ // - if the period we are looking for is finished, we look for a ts_archived that
216	+ // is greater than the last day of the archive
217	+ return $endTimestampUTC;
218	+ }
219	+
220	+ return false;
221	+ }
222	+
223	+ private function shouldArchiveForSiteEvenWhenNoVisits()
224	+ {
225	+ $idSitesToArchive = $this->getIdSitesToArchiveWhenNoVisits();
226	+ return in_array($this->params->getSite()->getId(), $idSitesToArchive);
227	+ }
228	+
229	+ private function getIdSitesToArchiveWhenNoVisits()
230	+ {
231	+ $cache = Cache::getTransientCache();
232	+ $cacheKey = 'Archiving.getIdSitesToArchiveWhenNoVisits';
233	+
234	+ if (!$cache->contains($cacheKey)) {
235	+ $idSites = array();
236	+
237	+ // leaving undocumented unless decided otherwise
238	+ Piwik::postEvent('Archiving.getIdSitesToArchiveWhenNoVisits', array(&$idSites));
239	+
240	+ $cache->save($cacheKey, $idSites);
241	+ }
242	+
243	+ return $cache->fetch($cacheKey);
244	+ }
245	+ }

app/core/ArchiveProcessor/Parameters.php ADDED Viewed

	@@ -0,0 +1,261 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+
10	+ namespace Piwik\ArchiveProcessor;
11	+
12	+ use Piwik\Date;
13	+ use Piwik\Log;
14	+ use Piwik\Period;
15	+ use Piwik\Piwik;
16	+ use Piwik\Segment;
17	+ use Piwik\Site;
18	+
19	+ /**
20	+ * Contains the analytics parameters for the reports that are currently being archived. The analytics
21	+ * parameters include the website the reports describe, the period of time the reports describe
22	+ * and the segment used to limit the visit set.
23	+ */
24	+ class Parameters
25	+ {
26	+ /**
27	+ * @var Site
28	+ */
29	+ private $site = null;
30	+
31	+ /**
32	+ * @var Period
33	+ */
34	+ private $period = null;
35	+
36	+ /**
37	+ * @var Segment
38	+ */
39	+ private $segment = null;
40	+
41	+ /**
42	+ * @var string Plugin name which triggered this archive processor
43	+ */
44	+ private $requestedPlugin = false;
45	+
46	+ private $onlyArchiveRequestedPlugin = false;
47	+
48	+ /**
49	+ * @var bool
50	+ */
51	+ private $isRootArchiveRequest = true;
52	+
53	+ /**
54	+ * Constructor.
55	+ *
56	+ * @ignore
57	+ */
58	+ public function __construct(Site $site, Period $period, Segment $segment)
59	+ {
60	+ $this->site = $site;
61	+ $this->period = $period;
62	+ $this->segment = $segment;
63	+ }
64	+
65	+ /**
66	+ * @ignore
67	+ */
68	+ public function setRequestedPlugin($plugin)
69	+ {
70	+ $this->requestedPlugin = $plugin;
71	+ }
72	+
73	+ /**
74	+ * @ignore
75	+ */
76	+ public function onlyArchiveRequestedPlugin()
77	+ {
78	+ $this->onlyArchiveRequestedPlugin = true;
79	+ }
80	+
81	+ /**
82	+ * @ignore
83	+ */
84	+ public function shouldOnlyArchiveRequestedPlugin()
85	+ {
86	+ return $this->onlyArchiveRequestedPlugin;
87	+ }
88	+
89	+ /**
90	+ * @ignore
91	+ */
92	+ public function getRequestedPlugin()
93	+ {
94	+ return $this->requestedPlugin;
95	+ }
96	+
97	+ /**
98	+ * Returns the period we are computing statistics for.
99	+ *
100	+ * @return Period
101	+ * @api
102	+ */
103	+ public function getPeriod()
104	+ {
105	+ return $this->period;
106	+ }
107	+
108	+ /**
109	+ * Returns the array of Period which make up this archive.
110	+ *
111	+ * @return \Piwik\Period[]
112	+ * @ignore
113	+ */
114	+ public function getSubPeriods()
115	+ {
116	+ if ($this->getPeriod()->getLabel() == 'day') {
117	+ return array( $this->getPeriod() );
118	+ }
119	+ return $this->getPeriod()->getSubperiods();
120	+ }
121	+
122	+ /**
123	+ * @return array
124	+ * @ignore
125	+ */
126	+ public function getIdSites()
127	+ {
128	+ $idSite = $this->getSite()->getId();
129	+
130	+ $idSites = array($idSite);
131	+
132	+ Piwik::postEvent('ArchiveProcessor.Parameters.getIdSites', array(&$idSites, $this->getPeriod()));
133	+
134	+ return $idSites;
135	+ }
136	+
137	+ /**
138	+ * Returns the site we are computing statistics for.
139	+ *
140	+ * @return Site
141	+ * @api
142	+ */
143	+ public function getSite()
144	+ {
145	+ return $this->site;
146	+ }
147	+
148	+ /**
149	+ * The Segment used to limit the set of visits that are being aggregated.
150	+ *
151	+ * @return Segment
152	+ * @api
153	+ */
154	+ public function getSegment()
155	+ {
156	+ return $this->segment;
157	+ }
158	+
159	+ /**
160	+ * Returns the end day of the period in the site's timezone.
161	+ *
162	+ * @return Date
163	+ */
164	+ public function getDateEnd()
165	+ {
166	+ return $this->getPeriod()->getDateEnd()->setTimezone($this->getSite()->getTimezone());
167	+ }
168	+
169	+ /**
170	+ * Returns the start day of the period in the site's timezone.
171	+ *
172	+ * @return Date
173	+ */
174	+ public function getDateStart()
175	+ {
176	+ return $this->getPeriod()->getDateStart()->setTimezone($this->getSite()->getTimezone());
177	+ }
178	+
179	+ /**
180	+ * Returns the start day of the period in the site's timezone (includes the time of day).
181	+ *
182	+ * @return Date
183	+ */
184	+ public function getDateTimeStart()
185	+ {
186	+ return $this->getPeriod()->getDateTimeStart()->setTimezone($this->getSite()->getTimezone());
187	+ }
188	+
189	+ /**
190	+ * Returns the end day of the period in the site's timezone (includes the time of day).
191	+ *
192	+ * @return Date
193	+ */
194	+ public function getDateTimeEnd()
195	+ {
196	+ return $this->getPeriod()->getDateTimeEnd()->setTimezone($this->getSite()->getTimezone());
197	+ }
198	+
199	+ /**
200	+ * @return bool
201	+ */
202	+ public function isSingleSiteDayArchive()
203	+ {
204	+ return $this->isDayArchive() && $this->isSingleSite();
205	+ }
206	+
207	+ /**
208	+ * @return bool
209	+ */
210	+ public function isDayArchive()
211	+ {
212	+ $period = $this->getPeriod();
213	+ $secondsInPeriod = $period->getDateEnd()->getTimestampUTC() - $period->getDateStart()->getTimestampUTC();
214	+ $oneDay = $secondsInPeriod < Date::NUM_SECONDS_IN_DAY;
215	+
216	+ return $oneDay;
217	+ }
218	+
219	+ public function isSingleSite()
220	+ {
221	+ return count($this->getIdSites()) == 1;
222	+ }
223	+
224	+ public function logStatusDebug()
225	+ {
226	+ $temporary = 'definitive archive';
227	+ Log::debug(
228	+ "%s archive, idSite = %d (%s), segment '%s', report = '%s', UTC datetime [%s -> %s]",
229	+ $this->getPeriod()->getLabel(),
230	+ $this->getSite()->getId(),
231	+ $temporary,
232	+ $this->getSegment()->getString(),
233	+ $this->getRequestedPlugin(),
234	+ $this->getDateStart()->getDateStartUTC(),
235	+ $this->getDateEnd()->getDateEndUTC()
236	+ );
237	+ }
238	+
239	+ /**
240	+ * Returns `true` if these parameters are part of an initial archiving request.
241	+ * Returns `false` if these parameters are for an archiving request that was initiated
242	+ * during archiving.
243	+ *
244	+ * @return bool
245	+ */
246	+ public function isRootArchiveRequest()
247	+ {
248	+ return $this->isRootArchiveRequest;
249	+ }
250	+
251	+ /**
252	+ * Sets whether these parameters are part of the initial archiving request or if they are
253	+ * for a request that was initiated during archiving.
254	+ *
255	+ * @param $isRootArchiveRequest
256	+ */
257	+ public function setIsRootArchiveRequest($isRootArchiveRequest)
258	+ {
259	+ $this->isRootArchiveRequest = $isRootArchiveRequest;
260	+ }
261	+ }

app/core/ArchiveProcessor/PluginsArchiver.php ADDED Viewed

	@@ -0,0 +1,339 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+
10	+ namespace Piwik\ArchiveProcessor;
11	+
12	+ use Piwik\ArchiveProcessor;
13	+ use Piwik\Container\StaticContainer;
14	+ use Piwik\CronArchive\Performance\Logger;
15	+ use Piwik\DataAccess\ArchiveWriter;
16	+ use Piwik\DataAccess\LogAggregator;
17	+ use Piwik\DataTable\Manager;
18	+ use Piwik\Metrics;
19	+ use Piwik\Piwik;
20	+ use Piwik\Plugin\Archiver;
21	+ use Piwik\Log;
22	+ use Piwik\Timer;
23	+ use Exception;
24	+
25	+ /**
26	+ * This class creates the Archiver objects found in plugins and will trigger aggregation,
27	+ * so each plugin can process their reports.
28	+ */
29	+ class PluginsArchiver
30	+ {
31	+ /**
32	+ * @var string\|null
33	+ */
34	+ private static $currentPluginBeingArchived = null;
35	+
36	+ /**
37	+ * @param ArchiveProcessor $archiveProcessor
38	+ */
39	+ public $archiveProcessor;
40	+
41	+ /**
42	+ * @var Parameters
43	+ */
44	+ protected $params;
45	+
46	+ /**
47	+ * @var LogAggregator
48	+ */
49	+ private $logAggregator;
50	+
51	+ /**
52	+ * Public only for tests. Won't be necessary after DI changes are complete.
53	+ *
54	+ * @var Archiver[] $archivers
55	+ */
56	+ public static $archivers = array();
57	+
58	+ /**
59	+ * Defines if we should aggregate from raw data by using MySQL queries (when true) or aggregate archives (when false)
60	+ * @var bool
61	+ */
62	+ private $shouldAggregateFromRawData;
63	+
64	+ public function __construct(Parameters $params, ArchiveWriter $archiveWriter = null)
65	+ {
66	+ $this->params = $params;
67	+ $this->archiveWriter = $archiveWriter ?: new ArchiveWriter($this->params);
68	+ $this->archiveWriter->initNewArchive();
69	+
70	+ $this->logAggregator = new LogAggregator($params);
71	+ $this->logAggregator->allowUsageSegmentCache();
72	+
73	+ $this->archiveProcessor = new ArchiveProcessor($this->params, $this->archiveWriter, $this->logAggregator);
74	+
75	+ $shouldAggregateFromRawData = $this->params->isSingleSiteDayArchive();
76	+
77	+ /**
78	+ * Triggered to detect if the archiver should aggregate from raw data by using MySQL queries (when true)
79	+ * or by aggregate archives (when false). Typically, data is aggregated from raw data for "day" period, and
80	+ * aggregregated from archives for all other periods.
81	+ *
82	+ * @param bool $shouldAggregateFromRawData Set to true, to aggregate from raw data, or false to aggregate multiple reports.
83	+ * @param Parameters $params
84	+ * @ignore
85	+ * @deprecated
86	+ *
87	+ * In Matomo 4.0 we should maybe remove this event, and instead maybe always archive from raw data when it is daily archive,
88	+ * no matter if single site or not. We cannot do this in Matomo 3.X as some custom plugin archivers may not be able to handle multiple sites.
89	+ */
90	+ Piwik::postEvent('ArchiveProcessor.shouldAggregateFromRawData', array(&$shouldAggregateFromRawData, $this->params));
91	+
92	+ $this->shouldAggregateFromRawData = $shouldAggregateFromRawData;
93	+ }
94	+
95	+ /**
96	+ * If period is day, will get the core metrics (including visits) from the logs.
97	+ * If period is != day, will sum the core metrics from the existing archives.
98	+ * @return array Core metrics
99	+ */
100	+ public function callAggregateCoreMetrics()
101	+ {
102	+ $this->logAggregator->cleanup();
103	+ $this->logAggregator->setQueryOriginHint('Core');
104	+
105	+ if ($this->shouldAggregateFromRawData) {
106	+ $metrics = $this->aggregateDayVisitsMetrics();
107	+ } else {
108	+ $metrics = $this->aggregateMultipleVisitsMetrics();
109	+ }
110	+
111	+ if (empty($metrics)) {
112	+ return array(
113	+ 'nb_visits' => false,
114	+ 'nb_visits_converted' => false
115	+ );
116	+ }
117	+ return array(
118	+ 'nb_visits' => $metrics['nb_visits'],
119	+ 'nb_visits_converted' => $metrics['nb_visits_converted']
120	+ );
121	+ }
122	+
123	+ /**
124	+ * Instantiates the Archiver class in each plugin that defines it,
125	+ * and triggers Aggregation processing on these plugins.
126	+ */
127	+ public function callAggregateAllPlugins($visits, $visitsConverted, $forceArchivingWithoutVisits = false)
128	+ {
129	+ Log::debug("PluginsArchiver::%s: Initializing archiving process for all plugins [visits = %s, visits converted = %s]",
130	+ __FUNCTION__, $visits, $visitsConverted);
131	+
132	+ /** @var Logger $performanceLogger */
133	+ $performanceLogger = StaticContainer::get(Logger::class);
134	+
135	+ $this->archiveProcessor->setNumberOfVisits($visits, $visitsConverted);
136	+
137	+ $archivers = static::getPluginArchivers();
138	+
139	+ foreach ($archivers as $pluginName => $archiverClass) {
140	+ // We clean up below all tables created during this function call (and recursive calls)
141	+ $latestUsedTableId = Manager::getInstance()->getMostRecentTableId();
142	+
143	+ /** @var Archiver $archiver */
144	+ $archiver = $this->makeNewArchiverObject($archiverClass, $pluginName);
145	+
146	+ if (!$archiver->isEnabled()) {
147	+ Log::debug("PluginsArchiver::%s: Skipping archiving for plugin '%s' (disabled).", __FUNCTION__, $pluginName);
148	+ continue;
149	+ }
150	+
151	+ if (!$forceArchivingWithoutVisits && !$visits && !$archiver->shouldRunEvenWhenNoVisits()) {
152	+ Log::debug("PluginsArchiver::%s: Skipping archiving for plugin '%s' (no visits).", __FUNCTION__, $pluginName);
153	+ continue;
154	+ }
155	+
156	+ if ($this->shouldProcessReportsForPlugin($pluginName)) {
157	+
158	+ $this->logAggregator->setQueryOriginHint($pluginName);
159	+
160	+ try {
161	+ self::$currentPluginBeingArchived = $pluginName;
162	+
163	+ $period = $this->params->getPeriod()->getLabel();
164	+
165	+ $timer = new Timer();
166	+ if ($this->shouldAggregateFromRawData) {
167	+ Log::debug("PluginsArchiver::%s: Archiving $period reports for plugin '%s' from raw data.", __FUNCTION__, $pluginName);
168	+
169	+ $archiver->callAggregateDayReport();
170	+ } else {
171	+ Log::debug("PluginsArchiver::%s: Archiving $period reports for plugin '%s' using reports for smaller periods.", __FUNCTION__, $pluginName);
172	+
173	+ $archiver->callAggregateMultipleReports();
174	+ }
175	+
176	+ $this->logAggregator->setQueryOriginHint('');
177	+
178	+ $performanceLogger->logMeasurement('plugin', $pluginName, $this->params, $timer);
179	+
180	+ Log::debug("PluginsArchiver::%s: %s while archiving %s reports for plugin '%s' %s.",
181	+ __FUNCTION__,
182	+ $timer->getMemoryLeak(),
183	+ $this->params->getPeriod()->getLabel(),
184	+ $pluginName,
185	+ $this->params->getSegment() ? sprintf("(for segment = '%s')", $this->params->getSegment()->getString()) : ''
186	+ );
187	+ } catch (Exception $e) {
188	+ throw new PluginsArchiverException($e->getMessage() . " - in plugin $pluginName", $e->getCode(), $e);
189	+ } finally {
190	+ self::$currentPluginBeingArchived = null;
191	+ }
192	+ } else {
193	+ Log::debug("PluginsArchiver::%s: Not archiving reports for plugin '%s'.", __FUNCTION__, $pluginName);
194	+ }
195	+
196	+ Manager::getInstance()->deleteAll($latestUsedTableId);
197	+ unset($archiver);
198	+ }
199	+
200	+ $this->logAggregator->cleanup();
201	+ }
202	+
203	+ public function finalizeArchive()
204	+ {
205	+ $this->params->logStatusDebug();
206	+ $this->archiveWriter->finalizeArchive();
207	+ $idArchive = $this->archiveWriter->getIdArchive();
208	+
209	+ return $idArchive;
210	+ }
211	+
212	+ /**
213	+ * Returns if any plugin archiver archives without visits
214	+ */
215	+ public static function doesAnyPluginArchiveWithoutVisits()
216	+ {
217	+ $archivers = static::getPluginArchivers();
218	+
219	+ foreach ($archivers as $pluginName => $archiverClass) {
220	+ if ($archiverClass::shouldRunEvenWhenNoVisits()) {
221	+ return true;
222	+ }
223	+ }
224	+
225	+ return false;
226	+ }
227	+
228	+ /**
229	+ * Loads Archiver class from any plugin that defines one.
230	+ *
231	+ * @return \Piwik\Plugin\Archiver[]
232	+ */
233	+ protected static function getPluginArchivers()
234	+ {
235	+ if (empty(static::$archivers)) {
236	+ $pluginNames = \Piwik\Plugin\Manager::getInstance()->getActivatedPlugins();
237	+ $archivers = array();
238	+ foreach ($pluginNames as $pluginName) {
239	+ $archivers[$pluginName] = self::getPluginArchiverClass($pluginName);
240	+ }
241	+ static::$archivers = array_filter($archivers);
242	+ }
243	+ return static::$archivers;
244	+ }
245	+
246	+ private static function getPluginArchiverClass($pluginName)
247	+ {
248	+ $klassName = 'Piwik\\Plugins\\' . $pluginName . '\\Archiver';
249	+ if (class_exists($klassName)
250	+ && is_subclass_of($klassName, 'Piwik\\Plugin\\Archiver')) {
251	+ return $klassName;
252	+ }
253	+ return false;
254	+ }
255	+
256	+ /**
257	+ * Whether the specified plugin's reports should be archived
258	+ * @param string $pluginName
259	+ * @return bool
260	+ */
261	+ protected function shouldProcessReportsForPlugin($pluginName)
262	+ {
263	+ if ($this->params->getRequestedPlugin() == $pluginName) {
264	+ return true;
265	+ }
266	+
267	+ if ($this->params->shouldOnlyArchiveRequestedPlugin()) {
268	+ return false;
269	+ }
270	+
271	+ if (Rules::shouldProcessReportsAllPlugins(
272	+ $this->params->getIdSites(),
273	+ $this->params->getSegment(),
274	+ $this->params->getPeriod()->getLabel())) {
275	+ return true;
276	+ }
277	+
278	+ if (!\Piwik\Plugin\Manager::getInstance()->isPluginLoaded($this->params->getRequestedPlugin())) {
279	+ return true;
280	+ }
281	+ return false;
282	+ }
283	+
284	+ protected function aggregateDayVisitsMetrics()
285	+ {
286	+ $query = $this->archiveProcessor->getLogAggregator()->queryVisitsByDimension();
287	+ $data = $query->fetch();
288	+
289	+ $metrics = $this->convertMetricsIdToName($data);
290	+ $this->archiveProcessor->insertNumericRecords($metrics);
291	+ return $metrics;
292	+ }
293	+
294	+ protected function convertMetricsIdToName($data)
295	+ {
296	+ $metrics = array();
297	+ foreach ($data as $metricId => $value) {
298	+ $readableMetric = Metrics::$mappingFromIdToName[$metricId];
299	+ $metrics[$readableMetric] = $value;
300	+ }
301	+ return $metrics;
302	+ }
303	+
304	+ protected function aggregateMultipleVisitsMetrics()
305	+ {
306	+ $toSum = Metrics::getVisitsMetricNames();
307	+ $metrics = $this->archiveProcessor->aggregateNumericMetrics($toSum);
308	+ return $metrics;
309	+ }
310	+
311	+
312	+ /**
313	+ * @param $archiverClass
314	+ * @return Archiver
315	+ */
316	+ private function makeNewArchiverObject($archiverClass, $pluginName)
317	+ {
318	+ $archiver = new $archiverClass($this->archiveProcessor);
319	+
320	+ /**
321	+ * Triggered right after a new plugin archiver instance is created.
322	+ * Subscribers to this event can configure the plugin archiver, for example prevent the archiving of a plugin's data
323	+ * by calling `$archiver->disable()` method.
324	+ *
325	+ * @param \Piwik\Plugin\Archiver &$archiver The newly created plugin archiver instance.
326	+ * @param string $pluginName The name of plugin of which archiver instance was created.
327	+ * @param array $this->params Array containing archive parameters (Site, Period, Date and Segment)
328	+ * @param bool false This parameter is deprecated and will be removed.
329	+ */
330	+ Piwik::postEvent('Archiving.makeNewArchiverObject', array($archiver, $pluginName, $this->params, false));
331	+
332	+ return $archiver;
333	+ }
334	+
335	+ public static function isArchivingProcessActive()
336	+ {
337	+ return self::$currentPluginBeingArchived !== null;
338	+ }
339	+ }

app/core/ArchiveProcessor/PluginsArchiverException.php ADDED Viewed

	@@ -0,0 +1,15 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+
10	+ namespace Piwik\ArchiveProcessor;
11	+
12	+
13	+ class PluginsArchiverException extends \Exception
14	+ {
15	+ }

app/core/ArchiveProcessor/Rules.php ADDED Viewed

	@@ -0,0 +1,321 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik\ArchiveProcessor;
10	+
11	+ use Exception;
12	+ use Piwik\Config;
13	+ use Piwik\DataAccess\ArchiveWriter;
14	+ use Piwik\Date;
15	+ use Piwik\Log;
16	+ use Piwik\Option;
17	+ use Piwik\Piwik;
18	+ use Piwik\Plugins\CoreAdminHome\Controller;
19	+ use Piwik\Segment;
20	+ use Piwik\SettingsPiwik;
21	+ use Piwik\SettingsServer;
22	+ use Piwik\Site;
23	+ use Piwik\Tracker\Cache;
24	+
25	+ /**
26	+ * This class contains Archiving rules/logic which are used when creating and processing Archives.
27	+ *
28	+ */
29	+ class Rules
30	+ {
31	+ const OPTION_TODAY_ARCHIVE_TTL = 'todayArchiveTimeToLive';
32	+
33	+ const OPTION_BROWSER_TRIGGER_ARCHIVING = 'enableBrowserTriggerArchiving';
34	+
35	+ const FLAG_TABLE_PURGED = 'lastPurge_';
36	+
37	+ /** Flag that will forcefully disable the archiving process (used in tests only) */
38	+ public static $archivingDisabledByTests = false;
39	+
40	+ /**
41	+ * Returns the name of the archive field used to tell the status of an archive, (ie,
42	+ * whether the archive was created successfully or not).
43	+ *
44	+ * @param array $idSites
45	+ * @param Segment $segment
46	+ * @param string $periodLabel
47	+ * @param string $plugin
48	+ * @return string
49	+ */
50	+ public static function getDoneStringFlagFor(array $idSites, $segment, $periodLabel, $plugin)
51	+ {
52	+ if (!self::shouldProcessReportsAllPlugins($idSites, $segment, $periodLabel)) {
53	+ return self::getDoneFlagArchiveContainsOnePlugin($segment, $plugin);
54	+ }
55	+ return self::getDoneFlagArchiveContainsAllPlugins($segment);
56	+ }
57	+
58	+ public static function shouldProcessReportsAllPlugins(array $idSites, Segment $segment, $periodLabel)
59	+ {
60	+ if ($segment->isEmpty() && ($periodLabel != 'range' \|\| SettingsServer::isArchivePhpTriggered())) {
61	+ return true;
62	+ }
63	+
64	+ return self::isSegmentPreProcessed($idSites, $segment);
65	+ }
66	+
67	+ /**
68	+ * @param $idSites
69	+ * @return array
70	+ */
71	+ public static function getSegmentsToProcess($idSites)
72	+ {
73	+ $knownSegmentsToArchiveAllSites = SettingsPiwik::getKnownSegmentsToArchive();
74	+
75	+ $segmentsToProcess = $knownSegmentsToArchiveAllSites;
76	+ foreach ($idSites as $idSite) {
77	+ $segmentForThisWebsite = SettingsPiwik::getKnownSegmentsToArchiveForSite($idSite);
78	+ $segmentsToProcess = array_merge($segmentsToProcess, $segmentForThisWebsite);
79	+ }
80	+ $segmentsToProcess = array_unique($segmentsToProcess);
81	+ return $segmentsToProcess;
82	+ }
83	+
84	+ public static function getDoneFlagArchiveContainsOnePlugin(Segment $segment, $plugin)
85	+ {
86	+ return 'done' . $segment->getHash() . '.' . $plugin ;
87	+ }
88	+
89	+ public static function getDoneFlagArchiveContainsAllPlugins(Segment $segment)
90	+ {
91	+ return 'done' . $segment->getHash();
92	+ }
93	+
94	+ /**
95	+ * Return done flags used to tell how the archiving process for a specific archive was completed,
96	+ *
97	+ * @param array $plugins
98	+ * @param $segment
99	+ * @return array
100	+ */
101	+ public static function getDoneFlags(array $plugins, Segment $segment)
102	+ {
103	+ $doneFlags = array();
104	+ $doneAllPlugins = self::getDoneFlagArchiveContainsAllPlugins($segment);
105	+ $doneFlags[$doneAllPlugins] = $doneAllPlugins;
106	+
107	+ $plugins = array_unique($plugins);
108	+ foreach ($plugins as $plugin) {
109	+ $doneOnePlugin = self::getDoneFlagArchiveContainsOnePlugin($segment, $plugin);
110	+ $doneFlags[$plugin] = $doneOnePlugin;
111	+ }
112	+ return $doneFlags;
113	+ }
114	+
115	+ public static function getMinTimeProcessedForInProgressArchive(
116	+ Date $dateStart, \Piwik\Period $period, Segment $segment, Site $site)
117	+ {
118	+ $todayArchiveTimeToLive = self::getPeriodArchiveTimeToLiveDefault($period->getLabel());
119	+
120	+ $now = time();
121	+ $minimumArchiveTime = $now - $todayArchiveTimeToLive;
122	+
123	+ $idSites = array($site->getId());
124	+ $isArchivingDisabled = Rules::isArchivingDisabledFor($idSites, $segment, $period->getLabel());
125	+ if ($isArchivingDisabled) {
126	+ if ($period->getNumberOfSubperiods() == 0
127	+ && $dateStart->getTimestamp() <= $now
128	+ ) {
129	+ // Today: accept any recent enough archive
130	+ $minimumArchiveTime = false;
131	+ } else {
132	+ // This week, this month, this year:
133	+ // accept any archive that was processed today after 00:00:01 this morning
134	+ $timezone = $site->getTimezone();
135	+ $minimumArchiveTime = Date::factory(Date::factory('now', $timezone)->getDateStartUTC())->setTimezone($timezone)->getTimestamp();
136	+ }
137	+ }
138	+ return $minimumArchiveTime;
139	+ }
140	+
141	+ public static function setTodayArchiveTimeToLive($timeToLiveSeconds)
142	+ {
143	+ $timeToLiveSeconds = (int)$timeToLiveSeconds;
144	+ if ($timeToLiveSeconds <= 0) {
145	+ throw new Exception(Piwik::translate('General_ExceptionInvalidArchiveTimeToLive'));
146	+ }
147	+ Option::set(self::OPTION_TODAY_ARCHIVE_TTL, $timeToLiveSeconds, $autoLoad = true);
148	+ }
149	+
150	+ public static function getTodayArchiveTimeToLive()
151	+ {
152	+ $uiSettingIsEnabled = Controller::isGeneralSettingsAdminEnabled();
153	+
154	+ if ($uiSettingIsEnabled) {
155	+ $timeToLive = Option::get(self::OPTION_TODAY_ARCHIVE_TTL);
156	+ if ($timeToLive !== false) {
157	+ return $timeToLive;
158	+ }
159	+ }
160	+ return self::getTodayArchiveTimeToLiveDefault();
161	+ }
162	+
163	+ public static function getPeriodArchiveTimeToLiveDefault($periodLabel)
164	+ {
165	+ if (empty($periodLabel) \|\| strtolower($periodLabel) === 'day') {
166	+ return self::getTodayArchiveTimeToLive();
167	+ }
168	+
169	+ $config = Config::getInstance();
170	+ $general = $config->General;
171	+
172	+ $key = sprintf('time_before_%s_archive_considered_outdated', $periodLabel);
173	+ if (isset($general[$key]) && is_numeric($general[$key]) && $general[$key] > 0) {
174	+ return $general[$key];
175	+ }
176	+
177	+ return self::getTodayArchiveTimeToLive();
178	+ }
179	+
180	+ public static function getTodayArchiveTimeToLiveDefault()
181	+ {
182	+ return Config::getInstance()->General['time_before_today_archive_considered_outdated'];
183	+ }
184	+
185	+ public static function isBrowserArchivingAvailableForSegments()
186	+ {
187	+ $generalConfig = Config::getInstance()->General;
188	+ return !$generalConfig['browser_archiving_disabled_enforce'];
189	+ }
190	+
191	+ public static function isArchivingDisabledFor(array $idSites, Segment $segment, $periodLabel)
192	+ {
193	+ $generalConfig = Config::getInstance()->General;
194	+
195	+ if ($periodLabel == 'range') {
196	+ if (!isset($generalConfig['archiving_range_force_on_browser_request'])
197	+ \|\| $generalConfig['archiving_range_force_on_browser_request'] != false
198	+ ) {
199	+ return false;
200	+ }
201	+
202	+ Log::debug("Not forcing archiving for range period.");
203	+ $processOneReportOnly = false;
204	+
205	+ } else {
206	+ $processOneReportOnly = !self::shouldProcessReportsAllPlugins($idSites, $segment, $periodLabel);
207	+ }
208	+
209	+ $isArchivingEnabled = self::isRequestAuthorizedToArchive() && !self::$archivingDisabledByTests;
210	+
211	+ if ($processOneReportOnly) {
212	+ // When there is a segment, we disable archiving when browser_archiving_disabled_enforce applies
213	+ if (!$segment->isEmpty()
214	+ && !$isArchivingEnabled
215	+ && !self::isBrowserArchivingAvailableForSegments()
216	+ && !SettingsServer::isArchivePhpTriggered() // Only applies when we are not running core:archive command
217	+ ) {
218	+ Log::debug("Archiving is disabled because of config setting browser_archiving_disabled_enforce=1");
219	+ return true;
220	+ }
221	+
222	+ // Always allow processing one report
223	+ return false;
224	+ }
225	+
226	+ return !$isArchivingEnabled;
227	+ }
228	+
229	+ public static function isRequestAuthorizedToArchive(Parameters $params = null)
230	+ {
231	+ $isRequestAuthorizedToArchive = Rules::isBrowserTriggerEnabled() \|\| SettingsServer::isArchivePhpTriggered();
232	+
233	+ if (!empty($params)) {
234	+ /**
235	+ * @ignore
236	+ *
237	+ * @params bool &$isRequestAuthorizedToArchive
238	+ * @params Parameters $params
239	+ */
240	+ Piwik::postEvent('Archiving.isRequestAuthorizedToArchive', [&$isRequestAuthorizedToArchive, $params]);
241	+ }
242	+
243	+ return $isRequestAuthorizedToArchive;
244	+ }
245	+
246	+ public static function isBrowserTriggerEnabled()
247	+ {
248	+ $uiSettingIsEnabled = Controller::isGeneralSettingsAdminEnabled();
249	+
250	+ if ($uiSettingIsEnabled) {
251	+ $browserArchivingEnabled = Option::get(self::OPTION_BROWSER_TRIGGER_ARCHIVING);
252	+ if ($browserArchivingEnabled !== false) {
253	+ return (bool)$browserArchivingEnabled;
254	+ }
255	+ }
256	+ return (bool)Config::getInstance()->General['enable_browser_archiving_triggering'];
257	+ }
258	+
259	+ public static function setBrowserTriggerArchiving($enabled)
260	+ {
261	+ if (!is_bool($enabled)) {
262	+ throw new Exception('Browser trigger archiving must be set to true or false.');
263	+ }
264	+ Option::set(self::OPTION_BROWSER_TRIGGER_ARCHIVING, (int)$enabled, $autoLoad = true);
265	+ Cache::clearCacheGeneral();
266	+ }
267	+
268	+ /**
269	+ * Returns true if the archiving process should skip the calculation of unique visitors
270	+ * across several sites. The `[General] enable_processing_unique_visitors_multiple_sites`
271	+ * INI config option controls the value of this variable.
272	+ *
273	+ * @return bool
274	+ */
275	+ public static function shouldSkipUniqueVisitorsCalculationForMultipleSites()
276	+ {
277	+ return Config::getInstance()->General['enable_processing_unique_visitors_multiple_sites'] != 1;
278	+ }
279	+
280	+ /**
281	+ * @param array $idSites
282	+ * @param Segment $segment
283	+ * @return bool
284	+ */
285	+ public static function isSegmentPreProcessed(array $idSites, Segment $segment)
286	+ {
287	+ $segmentsToProcess = self::getSegmentsToProcess($idSites);
288	+
289	+ if (empty($segmentsToProcess)) {
290	+ return false;
291	+ }
292	+ // If the requested segment is one of the segments to pre-process
293	+ // we ensure that any call to the API will trigger archiving of all reports for this segment
294	+ $segment = $segment->getString();
295	+
296	+ // Turns out the getString() above returns the URL decoded segment string
297	+ $segmentsToProcessUrlDecoded = array_map('urldecode', $segmentsToProcess);
298	+
299	+ return in_array($segment, $segmentsToProcess)
300	+ \|\| in_array($segment, $segmentsToProcessUrlDecoded);
301	+ }
302	+
303	+ /**
304	+ * Returns done flag values allowed to be selected
305	+ *
306	+ * @return string[]
307	+ */
308	+ public static function getSelectableDoneFlagValues($includeInvalidated = true, Parameters $params = null)
309	+ {
310	+ $possibleValues = array(ArchiveWriter::DONE_OK, ArchiveWriter::DONE_OK_TEMPORARY);
311	+
312	+ if (!Rules::isRequestAuthorizedToArchive($params)
313	+ && $includeInvalidated
314	+ ) {
315	+ //If request is not authorized to archive then fetch also invalidated archives
316	+ $possibleValues[] = ArchiveWriter::DONE_INVALIDATED;
317	+ }
318	+
319	+ return $possibleValues;
320	+ }
321	+ }

app/core/Archiver/Request.php ADDED Viewed

	@@ -0,0 +1,101 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik\Archiver;
10	+
11	+ class Request
12	+ {
13	+ /**
14	+ * If a request is aborted, the response of a CliMutli job will be a serialized array containing the
15	+ * key/value "aborted => 1".
16	+ */
17	+ const ABORT = 'abort';
18	+
19	+ /**
20	+ * @var string
21	+ */
22	+ private $url;
23	+
24	+ /**
25	+ * @var callable\|null
26	+ */
27	+ private $before;
28	+
29	+ /**
30	+ * @param string $url
31	+ */
32	+ public function __construct($url)
33	+ {
34	+ $this->setUrl($url);
35	+ }
36	+
37	+ public function before($callable)
38	+ {
39	+ $this->before = $callable;
40	+ }
41	+
42	+ public function start()
43	+ {
44	+ if ($this->before) {
45	+ return call_user_func($this->before);
46	+ }
47	+ }
48	+
49	+ public function __toString()
50	+ {
51	+ return $this->url;
52	+ }
53	+
54	+ /**
55	+ * @return string
56	+ */
57	+ public function getUrl()
58	+ {
59	+ return $this->url;
60	+ }
61	+
62	+ /**
63	+ * @param string $url
64	+ */
65	+ public function setUrl($url)
66	+ {
67	+ $this->url = $url;
68	+ }
69	+
70	+ public function changeDate($newDate)
71	+ {
72	+ $this->changeParam('date', $newDate);
73	+ }
74	+
75	+ public function makeSureDateIsNotSingleDayRange()
76	+ {
77	+ // TODO: revisit in matomo 4
78	+ // period=range&date=last1/period=range&date=previous1 can cause problems during archiving due to Parameters::isDayArchive()
79	+ if (preg_match('/[&?]period=range/', $this->url)) {
80	+ if (preg_match('/[&?]date=last1/', $this->url)) {
81	+ $this->changeParam('period', 'day');
82	+ $this->changeParam('date', 'today');
83	+ } else if (preg_match('/[&?]date=previous1/', $this->url)) {
84	+ $this->changeParam('period', 'day');
85	+ $this->changeParam('date', 'yesterday');
86	+ } else if (preg_match('/[&?]date=([^,]+),([^,&]+)/', $this->url, $matches)
87	+ && $matches[1] == $matches[2]
88	+ ) {
89	+ $this->changeParam('period', 'day');
90	+ $this->changeParam('date', $matches[1]);
91	+ }
92	+ }
93	+ }
94	+
95	+ public function changeParam($name, $newValue)
96	+ {
97	+ $url = $this->getUrl();
98	+ $url = preg_replace('/([&?])' . preg_quote($name) . '=[^&]*/', '$1' . $name . '=' . $newValue, $url);
99	+ $this->setUrl($url);
100	+ }
101	+ }

app/core/AssetManager.php ADDED Viewed

	@@ -0,0 +1,448 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik;
10	+
11	+ use Exception;
12	+ use Piwik\AssetManager\UIAsset;
13	+ use Piwik\AssetManager\UIAsset\InMemoryUIAsset;
14	+ use Piwik\AssetManager\UIAsset\OnDiskUIAsset;
15	+ use Piwik\AssetManager\UIAssetCacheBuster;
16	+ use Piwik\AssetManager\UIAssetFetcher\JScriptUIAssetFetcher;
17	+ use Piwik\AssetManager\UIAssetFetcher\StaticUIAssetFetcher;
18	+ use Piwik\AssetManager\UIAssetFetcher\StylesheetUIAssetFetcher;
19	+ use Piwik\AssetManager\UIAssetFetcher;
20	+ use Piwik\AssetManager\UIAssetMerger\JScriptUIAssetMerger;
21	+ use Piwik\AssetManager\UIAssetMerger\StylesheetUIAssetMerger;
22	+ use Piwik\Container\StaticContainer;
23	+ use Piwik\Plugin\Manager;
24	+
25	+ /**
26	+ * AssetManager is the class used to manage the inclusion of UI assets:
27	+ * JavaScript and CSS files.
28	+ *
29	+ * It performs the following actions:
30	+ * - Identifies required assets
31	+ * - Includes assets in the rendered HTML page
32	+ * - Manages asset merging and minifying
33	+ * - Manages server-side cache
34	+ *
35	+ * Whether assets are included individually or as merged files is defined by
36	+ * the global option 'disable_merged_assets'. See the documentation in the global
37	+ * config for more information.
38	+ */
39	+ class AssetManager extends Singleton
40	+ {
41	+ const MERGED_CSS_FILE = "asset_manager_global_css.css";
42	+ const MERGED_CORE_JS_FILE = "asset_manager_core_js.js";
43	+ const MERGED_NON_CORE_JS_FILE = "asset_manager_non_core_js.js";
44	+
45	+ const CSS_IMPORT_DIRECTIVE = "<link rel=\"stylesheet\" type=\"text/css\" href=\"%s\" />\n";
46	+ const JS_IMPORT_DIRECTIVE = "<script type=\"text/javascript\" src=\"%s\"></script>\n";
47	+ const GET_CSS_MODULE_ACTION = "index.php?module=Proxy&action=getCss";
48	+ const GET_CORE_JS_MODULE_ACTION = "index.php?module=Proxy&action=getCoreJs";
49	+ const GET_NON_CORE_JS_MODULE_ACTION = "index.php?module=Proxy&action=getNonCoreJs";
50	+
51	+ /**
52	+ * @var UIAssetCacheBuster
53	+ */
54	+ private $cacheBuster;
55	+
56	+ /**
57	+ * @var UIAssetFetcher
58	+ */
59	+ private $minimalStylesheetFetcher;
60	+
61	+ /**
62	+ * @var Theme
63	+ */
64	+ private $theme;
65	+
66	+ public function __construct()
67	+ {
68	+ $this->cacheBuster = UIAssetCacheBuster::getInstance();
69	+
70	+ $this->minimalStylesheetFetcher = new StaticUIAssetFetcher(array(), array(), $this->theme);
71	+
72	+ $theme = Manager::getInstance()->getThemeEnabled();
73	+ if (!empty($theme)) {
74	+ $this->theme = new Theme();
75	+ }
76	+ }
77	+
78	+ /**
79	+ * @inheritDoc
80	+ * @return AssetManager
81	+ */
82	+ public static function getInstance()
83	+ {
84	+ $assetManager = parent::getInstance();
85	+
86	+ /**
87	+ * Triggered when creating an instance of the asset manager. Lets you overwite the
88	+ * asset manager behavior.
89	+ *
90	+ * @param AssetManager &$assetManager
91	+ *
92	+ * @ignore
93	+ * This event is not a public event since we don't want to make the asset manager itself public
94	+ * API
95	+ */
96	+ Piwik::postEvent('AssetManager.makeNewAssetManagerObject', array(&$assetManager));
97	+
98	+ return $assetManager;
99	+ }
100	+
101	+ /**
102	+ * @param UIAssetCacheBuster $cacheBuster
103	+ */
104	+ public function setCacheBuster($cacheBuster)
105	+ {
106	+ $this->cacheBuster = $cacheBuster;
107	+ }
108	+
109	+ /**
110	+ * @param UIAssetFetcher $minimalStylesheetFetcher
111	+ */
112	+ public function setMinimalStylesheetFetcher($minimalStylesheetFetcher)
113	+ {
114	+ $this->minimalStylesheetFetcher = $minimalStylesheetFetcher;
115	+ }
116	+
117	+ /**
118	+ * @param Theme $theme
119	+ */
120	+ public function setTheme($theme)
121	+ {
122	+ $this->theme = $theme;
123	+ }
124	+
125	+ /**
126	+ * Return CSS file inclusion directive(s) using the markup <link>
127	+ *
128	+ * @return string
129	+ */
130	+ public function getCssInclusionDirective()
131	+ {
132	+ return sprintf(self::CSS_IMPORT_DIRECTIVE, self::GET_CSS_MODULE_ACTION);
133	+ }
134	+
135	+ /**
136	+ * Return JS file inclusion directive(s) using the markup <script>
137	+ *
138	+ * @return string
139	+ */
140	+ public function getJsInclusionDirective()
141	+ {
142	+ $result = "<script type=\"text/javascript\">\n" . Translate::getJavascriptTranslations() . "\n</script>";
143	+
144	+ if ($this->isMergedAssetsDisabled()) {
145	+ $this->getMergedCoreJSAsset()->delete();
146	+ $this->getMergedNonCoreJSAsset()->delete();
147	+
148	+ $result .= $this->getIndividualCoreAndNonCoreJsIncludes();
149	+ } else {
150	+ $result .= sprintf(self::JS_IMPORT_DIRECTIVE, self::GET_CORE_JS_MODULE_ACTION);
151	+ $result .= sprintf(self::JS_IMPORT_DIRECTIVE, self::GET_NON_CORE_JS_MODULE_ACTION);
152	+ }
153	+
154	+ return $result;
155	+ }
156	+
157	+ /**
158	+ * Return the base.less compiled to css
159	+ *
160	+ * @return UIAsset
161	+ */
162	+ public function getCompiledBaseCss()
163	+ {
164	+ $mergedAsset = new InMemoryUIAsset();
165	+
166	+ $assetMerger = new StylesheetUIAssetMerger($mergedAsset, $this->minimalStylesheetFetcher, $this->cacheBuster);
167	+
168	+ $assetMerger->generateFile();
169	+
170	+ return $mergedAsset;
171	+ }
172	+
173	+ /**
174	+ * Return the css merged file absolute location.
175	+ * If there is none, the generation process will be triggered.
176	+ *
177	+ * @return UIAsset
178	+ */
179	+ public function getMergedStylesheet()
180	+ {
181	+ $mergedAsset = $this->getMergedStylesheetAsset();
182	+
183	+ $assetFetcher = new StylesheetUIAssetFetcher(Manager::getInstance()->getLoadedPluginsName(), $this->theme);
184	+
185	+ $assetMerger = new StylesheetUIAssetMerger($mergedAsset, $assetFetcher, $this->cacheBuster);
186	+
187	+ $assetMerger->generateFile();
188	+
189	+ return $mergedAsset;
190	+ }
191	+
192	+ /**
193	+ * Return the core js merged file absolute location.
194	+ * If there is none, the generation process will be triggered.
195	+ *
196	+ * @return UIAsset
197	+ */
198	+ public function getMergedCoreJavaScript()
199	+ {
200	+ return $this->getMergedJavascript($this->getCoreJScriptFetcher(), $this->getMergedCoreJSAsset());
201	+ }
202	+
203	+ /**
204	+ * Return the non core js merged file absolute location.
205	+ * If there is none, the generation process will be triggered.
206	+ *
207	+ * @return UIAsset
208	+ */
209	+ public function getMergedNonCoreJavaScript()
210	+ {
211	+ return $this->getMergedJavascript($this->getNonCoreJScriptFetcher(), $this->getMergedNonCoreJSAsset());
212	+ }
213	+
214	+ /**
215	+ * @param boolean $core
216	+ * @return string[]
217	+ */
218	+ public function getLoadedPlugins($core)
219	+ {
220	+ $loadedPlugins = array();
221	+
222	+ foreach (Manager::getInstance()->getPluginsLoadedAndActivated() as $plugin) {
223	+ $pluginName = $plugin->getPluginName();
224	+ $pluginIsCore = Manager::getInstance()->isPluginBundledWithCore($pluginName);
225	+
226	+ if (($pluginIsCore && $core) \|\| (!$pluginIsCore && !$core)) {
227	+ $loadedPlugins[] = $pluginName;
228	+ }
229	+ }
230	+
231	+ return $loadedPlugins;
232	+ }
233	+
234	+ /**
235	+ * Remove previous merged assets
236	+ */
237	+ public function removeMergedAssets($pluginName = false)
238	+ {
239	+ $assetsToRemove = array($this->getMergedStylesheetAsset());
240	+
241	+ if ($pluginName) {
242	+ if ($this->pluginContainsJScriptAssets($pluginName)) {
243	+ if (Manager::getInstance()->isPluginBundledWithCore($pluginName)) {
244	+ $assetsToRemove[] = $this->getMergedCoreJSAsset();
245	+ } else {
246	+ $assetsToRemove[] = $this->getMergedNonCoreJSAsset();
247	+ }
248	+ }
249	+ } else {
250	+ $assetsToRemove[] = $this->getMergedCoreJSAsset();
251	+ $assetsToRemove[] = $this->getMergedNonCoreJSAsset();
252	+ }
253	+
254	+ $this->removeAssets($assetsToRemove);
255	+ }
256	+
257	+ /**
258	+ * Check if the merged file directory exists and is writable.
259	+ *
260	+ * @return string The directory location
261	+ * @throws Exception if directory is not writable.
262	+ */
263	+ public function getAssetDirectory()
264	+ {
265	+ $mergedFileDirectory = StaticContainer::get('path.tmp') . '/assets';
266	+
267	+ if (!is_dir($mergedFileDirectory)) {
268	+ Filesystem::mkdir($mergedFileDirectory);
269	+ }
270	+
271	+ if (!is_writable($mergedFileDirectory)) {
272	+ throw new Exception("Directory " . $mergedFileDirectory . " has to be writable.");
273	+ }
274	+
275	+ return $mergedFileDirectory;
276	+ }
277	+
278	+ /**
279	+ * Return the global option disable_merged_assets
280	+ *
281	+ * @return boolean
282	+ */
283	+ public function isMergedAssetsDisabled()
284	+ {
285	+ if (Config::getInstance()->Development['disable_merged_assets'] == 1) {
286	+ return true;
287	+ }
288	+
289	+ if (isset($_GET['disable_merged_assets']) && $_GET['disable_merged_assets'] == 1) {
290	+ return true;
291	+ }
292	+
293	+ return false;
294	+ }
295	+
296	+ /**
297	+ * @param UIAssetFetcher $assetFetcher
298	+ * @param UIAsset $mergedAsset
299	+ * @return UIAsset
300	+ */
301	+ private function getMergedJavascript($assetFetcher, $mergedAsset)
302	+ {
303	+ $assetMerger = new JScriptUIAssetMerger($mergedAsset, $assetFetcher, $this->cacheBuster);
304	+
305	+ $assetMerger->generateFile();
306	+
307	+ return $mergedAsset;
308	+ }
309	+
310	+ /**
311	+ * Return individual JS file inclusion directive(s) using the markup <script>
312	+ *
313	+ * @return string
314	+ */
315	+ protected function getIndividualCoreAndNonCoreJsIncludes()
316	+ {
317	+ return
318	+ $this->getIndividualJsIncludesFromAssetFetcher($this->getCoreJScriptFetcher()) .
319	+ $this->getIndividualJsIncludesFromAssetFetcher($this->getNonCoreJScriptFetcher());
320	+ }
321	+
322	+ /**
323	+ * @param UIAssetFetcher $assetFetcher
324	+ * @return string
325	+ */
326	+ protected function getIndividualJsIncludesFromAssetFetcher($assetFetcher)
327	+ {
328	+ $jsIncludeString = '';
329	+
330	+ $assets = $assetFetcher->getCatalog()->getAssets();
331	+
332	+ foreach ($assets as $jsFile) {
333	+ $jsFile->validateFile();
334	+ $jsIncludeString = $jsIncludeString . sprintf(self::JS_IMPORT_DIRECTIVE, $jsFile->getRelativeLocation());
335	+ }
336	+
337	+ return $jsIncludeString;
338	+ }
339	+
340	+ private function getCoreJScriptFetcher()
341	+ {
342	+ return new JScriptUIAssetFetcher($this->getLoadedPlugins(true), $this->theme);
343	+ }
344	+
345	+ protected function getNonCoreJScriptFetcher()
346	+ {
347	+ return new JScriptUIAssetFetcher($this->getLoadedPlugins(false), $this->theme);
348	+ }
349	+
350	+ /**
351	+ * @param string $pluginName
352	+ * @return boolean
353	+ */
354	+ private function pluginContainsJScriptAssets($pluginName)
355	+ {
356	+ $fetcher = new JScriptUIAssetFetcher(array($pluginName), $this->theme);
357	+
358	+ try {
359	+ $assets = $fetcher->getCatalog()->getAssets();
360	+ } catch (\Exception $e) {
361	+ // This can happen when a plugin is not valid (eg. Piwik 1.x format)
362	+ // When posting the event to the plugin, it returns an exception "Plugin has not been loaded"
363	+ return false;
364	+ }
365	+
366	+ $pluginManager = Manager::getInstance();
367	+ $plugin = $pluginManager->getLoadedPlugin($pluginName);
368	+
369	+ if ($plugin->isTheme()) {
370	+ $theme = $pluginManager->getTheme($pluginName);
371	+
372	+ $javaScriptFiles = $theme->getJavaScriptFiles();
373	+
374	+ if (!empty($javaScriptFiles)) {
375	+ $assets = array_merge($assets, $javaScriptFiles);
376	+ }
377	+ }
378	+
379	+ return !empty($assets);
380	+ }
381	+
382	+ /**
383	+ * @param UIAsset[] $uiAssets
384	+ */
385	+ public function removeAssets($uiAssets)
386	+ {
387	+ foreach ($uiAssets as $uiAsset) {
388	+ $uiAsset->delete();
389	+ }
390	+ }
391	+
392	+ /**
393	+ * @return UIAsset
394	+ */
395	+ public function getMergedStylesheetAsset()
396	+ {
397	+ return $this->getMergedUIAsset(self::MERGED_CSS_FILE);
398	+ }
399	+
400	+ /**
401	+ * @return UIAsset
402	+ */
403	+ private function getMergedCoreJSAsset()
404	+ {
405	+ return $this->getMergedUIAsset(self::MERGED_CORE_JS_FILE);
406	+ }
407	+
408	+ /**
409	+ * @return UIAsset
410	+ */
411	+ protected function getMergedNonCoreJSAsset()
412	+ {
413	+ return $this->getMergedUIAsset(self::MERGED_NON_CORE_JS_FILE);
414	+ }
415	+
416	+ /**
417	+ * @param string $fileName
418	+ * @return UIAsset
419	+ */
420	+ private function getMergedUIAsset($fileName)
421	+ {
422	+ return new OnDiskUIAsset($this->getAssetDirectory(), $fileName);
423	+ }
424	+
425	+ public static function compileCustomStylesheets($files)
426	+ {
427	+ $assetManager = new AssetManager();
428	+
429	+ $fetcher = new StaticUIAssetFetcher($files, $priorityOrder = array(), $theme = null);
430	+
431	+ $assetManager->setMinimalStylesheetFetcher($fetcher);
432	+
433	+ return $assetManager->getCompiledBaseCss()->getContent();
434	+ }
435	+
436	+ public static function compileCustomJs($files)
437	+ {
438	+ $mergedAsset = new InMemoryUIAsset();
439	+ $fetcher = new StaticUIAssetFetcher($files, $priorityOrder = array(), $theme = null);
440	+
441	+ $cacheBuster = UIAssetCacheBuster::getInstance();
442	+
443	+ $assetMerger = new JScriptUIAssetMerger($mergedAsset, $fetcher, $cacheBuster);
444	+ $assetMerger->generateFile();
445	+
446	+ return $mergedAsset->getContent();
447	+ }
448	+ }

app/core/AssetManager/UIAsset.php ADDED Viewed

	@@ -0,0 +1,61 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik\AssetManager;
10	+
11	+ use Exception;
12	+
13	+ abstract class UIAsset
14	+ {
15	+ abstract public function validateFile();
16	+
17	+ /**
18	+ * @return string
19	+ */
20	+ abstract public function getAbsoluteLocation();
21	+
22	+ /**
23	+ * @return string
24	+ */
25	+ abstract public function getRelativeLocation();
26	+
27	+ /**
28	+ * @return string
29	+ */
30	+ abstract public function getBaseDirectory();
31	+
32	+ /**
33	+ * Removes the previous file if it exists.
34	+ * Also tries to remove compressed version of the file.
35	+ *
36	+ * @see ProxyStaticFile::serveStaticFile(serveFile
37	+ * @throws Exception if the file couldn't be deleted
38	+ */
39	+ abstract public function delete();
40	+
41	+ /**
42	+ * @param string $content
43	+ * @throws \Exception
44	+ */
45	+ abstract public function writeContent($content);
46	+
47	+ /**
48	+ * @return string
49	+ */
50	+ abstract public function getContent();
51	+
52	+ /**
53	+ * @return boolean
54	+ */
55	+ abstract public function exists();
56	+
57	+ /**
58	+ * @return int
59	+ */
60	+ abstract public function getModificationDate();
61	+ }

app/core/AssetManager/UIAsset/InMemoryUIAsset.php ADDED Viewed

	@@ -0,0 +1,62 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik\AssetManager\UIAsset;
10	+
11	+ use Exception;
12	+ use Piwik\AssetManager\UIAsset;
13	+
14	+ class InMemoryUIAsset extends UIAsset
15	+ {
16	+ private $content;
17	+
18	+ public function validateFile()
19	+ {
20	+ return;
21	+ }
22	+
23	+ public function getAbsoluteLocation()
24	+ {
25	+ throw new Exception('invalid operation');
26	+ }
27	+
28	+ public function getRelativeLocation()
29	+ {
30	+ throw new Exception('invalid operation');
31	+ }
32	+
33	+ public function getBaseDirectory()
34	+ {
35	+ throw new Exception('invalid operation');
36	+ }
37	+
38	+ public function delete()
39	+ {
40	+ $this->content = null;
41	+ }
42	+
43	+ public function exists()
44	+ {
45	+ return false;
46	+ }
47	+
48	+ public function writeContent($content)
49	+ {
50	+ $this->content = $content;
51	+ }
52	+
53	+ public function getContent()
54	+ {
55	+ return $this->content;
56	+ }
57	+
58	+ public function getModificationDate()
59	+ {
60	+ throw new Exception('invalid operation');
61	+ }
62	+ }

app/core/AssetManager/UIAsset/OnDiskUIAsset.php ADDED Viewed

	@@ -0,0 +1,135 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik\AssetManager\UIAsset;
10	+
11	+ use Exception;
12	+ use Piwik\AssetManager\UIAsset;
13	+ use Piwik\Common;
14	+ use Piwik\Filesystem;
15	+
16	+ class OnDiskUIAsset extends UIAsset
17	+ {
18	+ /**
19	+ * @var string
20	+ */
21	+ private $baseDirectory;
22	+
23	+ /**
24	+ * @var string
25	+ */
26	+ private $relativeLocation;
27	+
28	+ /**
29	+ * @var string
30	+ */
31	+ private $relativeRootDir;
32	+
33	+ /**
34	+ * @param string $baseDirectory
35	+ * @param string $fileLocation
36	+ */
37	+ public function __construct($baseDirectory, $fileLocation, $relativeRootDir = '')
38	+ {
39	+ $this->baseDirectory = $baseDirectory;
40	+ $this->relativeLocation = $fileLocation;
41	+
42	+ if (!empty($relativeRootDir)
43	+ && is_string($relativeRootDir)
44	+ && !Common::stringEndsWith($relativeRootDir, '/')) {
45	+ $relativeRootDir .= '/';
46	+ }
47	+
48	+ $this->relativeRootDir = $relativeRootDir;
49	+ }
50	+
51	+ public function getAbsoluteLocation()
52	+ {
53	+ return $this->baseDirectory . '/' . $this->relativeLocation;
54	+ }
55	+
56	+ public function getRelativeLocation()
57	+ {
58	+ if (isset($this->relativeRootDir)) {
59	+ return $this->relativeRootDir . $this->relativeLocation;
60	+ }
61	+ return $this->relativeLocation;
62	+ }
63	+
64	+ public function getBaseDirectory()
65	+ {
66	+ return $this->baseDirectory;
67	+ }
68	+
69	+ public function validateFile()
70	+ {
71	+ if (!$this->assetIsReadable()) {
72	+ throw new Exception("The ui asset with 'href' = " . $this->getAbsoluteLocation() . " is not readable");
73	+ }
74	+ }
75	+
76	+ public function delete()
77	+ {
78	+ if ($this->exists()) {
79	+ try {
80	+ Filesystem::remove($this->getAbsoluteLocation());
81	+ } catch (Exception $e) {
82	+ throw new Exception("Unable to delete merged file : " . $this->getAbsoluteLocation() . ". Please delete the file and refresh");
83	+ }
84	+
85	+ // try to remove compressed version of the merged file.
86	+ Filesystem::remove($this->getAbsoluteLocation() . ".deflate", true);
87	+ Filesystem::remove($this->getAbsoluteLocation() . ".gz", true);
88	+ }
89	+ }
90	+
91	+ /**
92	+ * @param string $content
93	+ * @throws \Exception
94	+ */
95	+ public function writeContent($content)
96	+ {
97	+ $this->delete();
98	+
99	+ $newFile = @fopen($this->getAbsoluteLocation(), "w");
100	+
101	+ if (!$newFile) {
102	+ throw new Exception("The file : " . $newFile . " can not be opened in write mode.");
103	+ }
104	+
105	+ fwrite($newFile, $content);
106	+
107	+ fclose($newFile);
108	+ }
109	+
110	+ /**
111	+ * @return string
112	+ */
113	+ public function getContent()
114	+ {
115	+ return file_get_contents($this->getAbsoluteLocation());
116	+ }
117	+
118	+ public function exists()
119	+ {
120	+ return $this->assetIsReadable();
121	+ }
122	+
123	+ /**
124	+ * @return boolean
125	+ */
126	+ private function assetIsReadable()
127	+ {
128	+ return is_readable($this->getAbsoluteLocation());
129	+ }
130	+
131	+ public function getModificationDate()
132	+ {
133	+ return filemtime($this->getAbsoluteLocation());
134	+ }
135	+ }

app/core/AssetManager/UIAssetCacheBuster.php ADDED Viewed

	@@ -0,0 +1,66 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ * @method static \Piwik\AssetManager\UIAssetCacheBuster getInstance()
9	+ */
10	+ namespace Piwik\AssetManager;
11	+
12	+ use Piwik\Plugin\Manager;
13	+ use Piwik\Singleton;
14	+ use Piwik\Version;
15	+
16	+ class UIAssetCacheBuster extends Singleton
17	+ {
18	+ /**
19	+ * Cache buster based on
20	+ * - Piwik version
21	+ * - Loaded plugins (name and version)
22	+ * - Super user salt
23	+ * - Latest
24	+ *
25	+ * @param string[] $pluginNames
26	+ * @return string
27	+ */
28	+ public function piwikVersionBasedCacheBuster($pluginNames = false)
29	+ {
30	+ static $cachedCacheBuster = null;
31	+
32	+ if (empty($cachedCacheBuster) \|\| $pluginNames !== false) {
33	+
34	+ $masterFile = PIWIK_INCLUDE_PATH . '/.git/refs/heads/master';
35	+ $currentGitHash = file_exists($masterFile) ? @file_get_contents($masterFile) : null;
36	+
37	+ $plugins = !$pluginNames ? Manager::getInstance()->getLoadedPluginsName() : $pluginNames;
38	+ sort($plugins);
39	+
40	+ $pluginsInfo = '';
41	+ foreach ($plugins as $pluginName) {
42	+ $plugin = Manager::getInstance()->getLoadedPlugin($pluginName);
43	+ $pluginsInfo .= $plugin->getPluginName() . $plugin->getVersion() . ',';
44	+ }
45	+
46	+ $cacheBuster = md5($pluginsInfo . PHP_VERSION . Version::VERSION . trim($currentGitHash));
47	+
48	+ if ($pluginNames !== false) {
49	+ return $cacheBuster;
50	+ }
51	+
52	+ $cachedCacheBuster = $cacheBuster;
53	+ }
54	+
55	+ return $cachedCacheBuster;
56	+ }
57	+
58	+ /**
59	+ * @param string $content
60	+ * @return string
61	+ */
62	+ public function md5BasedCacheBuster($content)
63	+ {
64	+ return md5($content);
65	+ }
66	+ }

app/core/AssetManager/UIAssetCatalog.php ADDED Viewed

	@@ -0,0 +1,73 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik\AssetManager;
10	+
11	+ class UIAssetCatalog
12	+ {
13	+ /**
14	+ * @var UIAsset[]
15	+ */
16	+ private $uiAssets = array();
17	+
18	+ /**
19	+ * @var UIAssetCatalogSorter
20	+ */
21	+ private $catalogSorter;
22	+
23	+ /**
24	+ * @var string[] Absolute file locations
25	+ */
26	+ private $existingAssetLocations = array();
27	+
28	+ /**
29	+ * @param UIAssetCatalogSorter $catalogSorter
30	+ */
31	+ public function __construct($catalogSorter)
32	+ {
33	+ $this->catalogSorter = $catalogSorter;
34	+ }
35	+
36	+ /**
37	+ * @param UIAsset $uiAsset
38	+ */
39	+ public function addUIAsset($uiAsset)
40	+ {
41	+ $location = $uiAsset->getAbsoluteLocation();
42	+
43	+ if (!$this->assetAlreadyInCatalog($location)) {
44	+ $this->existingAssetLocations[] = $location;
45	+ $this->uiAssets[] = $uiAsset;
46	+ }
47	+ }
48	+
49	+ /**
50	+ * @return UIAsset[]
51	+ */
52	+ public function getAssets()
53	+ {
54	+ return $this->uiAssets;
55	+ }
56	+
57	+ /**
58	+ * @return UIAssetCatalog
59	+ */
60	+ public function getSortedCatalog()
61	+ {
62	+ return $this->catalogSorter->sortUIAssetCatalog($this);
63	+ }
64	+
65	+ /**
66	+ * @param UIAsset $uiAsset
67	+ * @return boolean
68	+ */
69	+ private function assetAlreadyInCatalog($location)
70	+ {
71	+ return in_array($location, $this->existingAssetLocations);
72	+ }
73	+ }

app/core/AssetManager/UIAssetCatalogSorter.php ADDED Viewed

	@@ -0,0 +1,58 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik\AssetManager;
10	+
11	+ class UIAssetCatalogSorter
12	+ {
13	+ /**
14	+ * @var string[]
15	+ */
16	+ private $priorityOrder;
17	+
18	+ /**
19	+ * @param string[] $priorityOrder
20	+ */
21	+ public function __construct($priorityOrder)
22	+ {
23	+ $this->priorityOrder = $priorityOrder;
24	+ }
25	+
26	+ /**
27	+ * @param UIAssetCatalog $uiAssetCatalog
28	+ * @return UIAssetCatalog
29	+ */
30	+ public function sortUIAssetCatalog($uiAssetCatalog)
31	+ {
32	+ $sortedCatalog = new UIAssetCatalog($this);
33	+ foreach ($this->priorityOrder as $filePattern) {
34	+ $assetsMatchingPattern = array_filter($uiAssetCatalog->getAssets(), function ($uiAsset) use ($filePattern) {
35	+ return preg_match('~^' . $filePattern . '~', $uiAsset->getRelativeLocation());
36	+ });
37	+
38	+ foreach ($assetsMatchingPattern as $assetMatchingPattern) {
39	+ $sortedCatalog->addUIAsset($assetMatchingPattern);
40	+ }
41	+ }
42	+
43	+ $this->addUnmatchedAssets($uiAssetCatalog, $sortedCatalog);
44	+
45	+ return $sortedCatalog;
46	+ }
47	+
48	+ /**
49	+ * @param UIAssetCatalog $uiAssetCatalog
50	+ * @param UIAssetCatalog $sortedCatalog
51	+ */
52	+ private function addUnmatchedAssets($uiAssetCatalog, $sortedCatalog)
53	+ {
54	+ foreach ($uiAssetCatalog->getAssets() as $uiAsset) {
55	+ $sortedCatalog->addUIAsset($uiAsset);
56	+ }
57	+ }
58	+ }

app/core/AssetManager/UIAssetFetcher.php ADDED Viewed

	@@ -0,0 +1,150 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik\AssetManager;
10	+
11	+ use Piwik\AssetManager\UIAsset\OnDiskUIAsset;
12	+ use Piwik\Plugin\Manager;
13	+ use Piwik\Theme;
14	+
15	+ abstract class UIAssetFetcher
16	+ {
17	+ /**
18	+ * @var UIAssetCatalog
19	+ */
20	+ protected $catalog;
21	+
22	+ /**
23	+ * @var string[]
24	+ */
25	+ protected $fileLocations = array();
26	+
27	+ /**
28	+ * @var string[]
29	+ */
30	+ protected $plugins;
31	+
32	+ /**
33	+ * @var Theme
34	+ */
35	+ private $theme;
36	+
37	+ /**
38	+ * @param string[] $plugins
39	+ * @param Theme $theme
40	+ */
41	+ public function __construct($plugins, $theme)
42	+ {
43	+ $this->plugins = $plugins;
44	+ $this->theme = $theme;
45	+ }
46	+
47	+ /**
48	+ * @return string[]
49	+ */
50	+ public function getPlugins()
51	+ {
52	+ return $this->plugins;
53	+ }
54	+
55	+ /**
56	+ * $return UIAssetCatalog
57	+ */
58	+ public function getCatalog()
59	+ {
60	+ if ($this->catalog == null) {
61	+ $this->createCatalog();
62	+ }
63	+
64	+ return $this->catalog;
65	+ }
66	+
67	+ abstract protected function retrieveFileLocations();
68	+
69	+ /**
70	+ * @return string[]
71	+ */
72	+ abstract protected function getPriorityOrder();
73	+
74	+ private function createCatalog()
75	+ {
76	+ $this->retrieveFileLocations();
77	+
78	+ $this->initCatalog();
79	+
80	+ $this->populateCatalog();
81	+
82	+ $this->sortCatalog();
83	+ }
84	+
85	+ private function initCatalog()
86	+ {
87	+ $catalogSorter = new UIAssetCatalogSorter($this->getPriorityOrder());
88	+ $this->catalog = new UIAssetCatalog($catalogSorter);
89	+ }
90	+
91	+ private function populateCatalog()
92	+ {
93	+ $pluginBaseDir = Manager::getPluginsDirectory();
94	+ $pluginWebDirectories = Manager::getAlternativeWebRootDirectories();
95	+ $matomoRootDir = $this->getBaseDirectory();
96	+
97	+ foreach ($this->fileLocations as $fileLocation) {
98	+ $fileAbsolute = $matomoRootDir . '/' . $fileLocation;
99	+
100	+ $newUIAsset = new OnDiskUIAsset($this->getBaseDirectory(), $fileLocation);
101	+ if ($newUIAsset->exists()) {
102	+ $this->catalog->addUIAsset($newUIAsset);
103	+ continue;
104	+ }
105	+
106	+ $found = false;
107	+
108	+ if (strpos($fileAbsolute, $pluginBaseDir) === 0) {
109	+ // we iterate over all custom plugin directories only for plugin files, not libs files (not needed there)
110	+ foreach ($pluginWebDirectories as $pluginDirectory => $relative) {
111	+ $fileTest = str_replace($pluginBaseDir, $pluginDirectory, $fileAbsolute);
112	+ $newFileRelative = str_replace($pluginDirectory, '', $fileTest);
113	+ $testAsset = new OnDiskUIAsset($pluginDirectory, $newFileRelative, $relative);
114	+ if ($testAsset->exists()) {
115	+ $this->catalog->addUIAsset($testAsset);
116	+ $found = true;
117	+ break;
118	+ }
119	+ }
120	+ }
121	+
122	+ if (!$found) {
123	+ // we add it anyway so it'll trigger an error about the missing file
124	+ $this->catalog->addUIAsset($newUIAsset);
125	+ }
126	+ }
127	+ }
128	+
129	+ private function sortCatalog()
130	+ {
131	+ $this->catalog = $this->catalog->getSortedCatalog();
132	+ }
133	+
134	+ /**
135	+ * @return string
136	+ */
137	+ private function getBaseDirectory()
138	+ {
139	+ // served by web server directly, so must be a public path
140	+ return PIWIK_DOCUMENT_ROOT;
141	+ }
142	+
143	+ /**
144	+ * @return Theme
145	+ */
146	+ public function getTheme()
147	+ {
148	+ return $this->theme;
149	+ }
150	+ }

app/core/AssetManager/UIAssetFetcher/JScriptUIAssetFetcher.php ADDED Viewed

	@@ -0,0 +1,88 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik\AssetManager\UIAssetFetcher;
10	+
11	+ use Piwik\AssetManager\UIAssetFetcher;
12	+ use Piwik\Piwik;
13	+
14	+ class JScriptUIAssetFetcher extends UIAssetFetcher
15	+ {
16	+
17	+ protected function retrieveFileLocations()
18	+ {
19	+ if (!empty($this->plugins)) {
20	+
21	+ /**
22	+ * Triggered when gathering the list of all JavaScript files needed by Piwik
23	+ * and its plugins.
24	+ *
25	+ * Plugins that have their own JavaScript should use this event to make those
26	+ * files load in the browser.
27	+ *
28	+ * JavaScript files should be placed within a javascripts subdirectory in your
29	+ * plugin's root directory.
30	+ *
31	+ * _Note: While you are developing your plugin you should enable the config setting
32	+ * `[Development] disable_merged_assets` so JavaScript files will be reloaded immediately
33	+ * after every change._
34	+ *
35	+ * Example
36	+ *
37	+ * public function getJsFiles(&$jsFiles)
38	+ * {
39	+ * $jsFiles[] = "plugins/MyPlugin/javascripts/myfile.js";
40	+ * $jsFiles[] = "plugins/MyPlugin/javascripts/anotherone.js";
41	+ * }
42	+ *
43	+ * @param string[] $jsFiles The JavaScript files to load.
44	+ */
45	+ Piwik::postEvent('AssetManager.getJavaScriptFiles', array(&$this->fileLocations), null, $this->plugins);
46	+ }
47	+
48	+ $this->addThemeFiles();
49	+ }
50	+
51	+ protected function addThemeFiles()
52	+ {
53	+ $theme = $this->getTheme();
54	+ if (!$theme) {
55	+ return;
56	+ }
57	+ if (in_array($theme->getThemeName(), $this->plugins)) {
58	+ $jsInThemes = $this->getTheme()->getJavaScriptFiles();
59	+
60	+ if (!empty($jsInThemes)) {
61	+ foreach ($jsInThemes as $jsFile) {
62	+ $this->fileLocations[] = $jsFile;
63	+ }
64	+ }
65	+ }
66	+ }
67	+
68	+ protected function getPriorityOrder()
69	+ {
70	+ return array(
71	+ 'libs/bower_components/jquery/dist/jquery.min.js',
72	+ 'libs/bower_components/jquery-ui/ui/minified/jquery-ui.min.js',
73	+ 'libs/jquery/jquery.browser.js',
74	+ 'libs/',
75	+ 'js/',
76	+ 'piwik.js',
77	+ 'matomo.js',
78	+ 'plugins/CoreHome/javascripts/require.js',
79	+ 'plugins/Morpheus/javascripts/piwikHelper.js',
80	+ 'plugins/Morpheus/javascripts/',
81	+ 'plugins/CoreHome/javascripts/uiControl.js',
82	+ 'plugins/CoreHome/javascripts/broadcast.js',
83	+ 'plugins/CoreHome/javascripts/', // load CoreHome JS before other plugins
84	+ 'plugins/',
85	+ 'tests/',
86	+ );
87	+ }
88	+ }

app/core/AssetManager/UIAssetFetcher/StaticUIAssetFetcher.php ADDED Viewed

	@@ -0,0 +1,36 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik\AssetManager\UIAssetFetcher;
10	+
11	+ use Piwik\AssetManager\UIAssetFetcher;
12	+
13	+ class StaticUIAssetFetcher extends UIAssetFetcher
14	+ {
15	+ /**
16	+ * @var string[]
17	+ */
18	+ private $priorityOrder;
19	+
20	+ public function __construct($fileLocations, $priorityOrder, $theme)
21	+ {
22	+ parent::__construct(array(), $theme);
23	+
24	+ $this->fileLocations = $fileLocations;
25	+ $this->priorityOrder = $priorityOrder;
26	+ }
27	+
28	+ protected function retrieveFileLocations()
29	+ {
30	+ }
31	+
32	+ protected function getPriorityOrder()
33	+ {
34	+ return $this->priorityOrder;
35	+ }
36	+ }

app/core/AssetManager/UIAssetFetcher/StylesheetUIAssetFetcher.php ADDED Viewed

	@@ -0,0 +1,85 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik\AssetManager\UIAssetFetcher;
10	+
11	+ use Piwik\AssetManager\UIAssetFetcher;
12	+ use Piwik\Piwik;
13	+
14	+ class StylesheetUIAssetFetcher extends UIAssetFetcher
15	+ {
16	+ protected function getPriorityOrder()
17	+ {
18	+ $theme = $this->getTheme();
19	+ $themeName = $theme->getThemeName();
20	+
21	+ $order = array(
22	+ 'plugins/Morpheus/stylesheets/base/bootstrap.css',
23	+ 'plugins/Morpheus/stylesheets/base/icons.css',
24	+ 'libs/',
25	+ 'plugins/CoreHome/stylesheets/color_manager.css', // must be before other Piwik stylesheets
26	+ 'plugins/Morpheus/stylesheets/base.less',
27	+ );
28	+
29	+ if ($themeName === 'Morpheus') {
30	+ $order[] = 'plugins\/((?!Morpheus).)*\/';
31	+ } else {
32	+ $order[] = sprintf('plugins\/((?!(Morpheus)\|(%s)).)*\/', $themeName);
33	+ }
34	+
35	+ $order = array_merge(
36	+ $order,
37	+ array(
38	+ 'plugins/Dashboard/stylesheets/dashboard.less',
39	+ 'tests/',
40	+ )
41	+ );
42	+
43	+ return $order;
44	+ }
45	+
46	+ protected function retrieveFileLocations()
47	+ {
48	+ /**
49	+ * Triggered when gathering the list of all stylesheets (CSS and LESS) needed by
50	+ * Piwik and its plugins.
51	+ *
52	+ * Plugins that have stylesheets should use this event to make those stylesheets
53	+ * load.
54	+ *
55	+ * Stylesheets should be placed within a stylesheets subdirectory in your plugin's
56	+ * root directory.
57	+ *
58	+ * Example
59	+ *
60	+ * public function getStylesheetFiles(&$stylesheets)
61	+ * {
62	+ * $stylesheets[] = "plugins/MyPlugin/stylesheets/myfile.less";
63	+ * $stylesheets[] = "plugins/MyPlugin/stylesheets/myotherfile.css";
64	+ * }
65	+ *
66	+ * @param string[] &$stylesheets The list of stylesheet paths.
67	+ */
68	+ Piwik::postEvent('AssetManager.getStylesheetFiles', array(&$this->fileLocations));
69	+
70	+ $this->addThemeFiles();
71	+ }
72	+
73	+ protected function addThemeFiles()
74	+ {
75	+ $theme = $this->getTheme();
76	+ if (!$theme) {
77	+ return;
78	+ }
79	+ $themeStylesheet = $theme->getStylesheet();
80	+
81	+ if ($themeStylesheet) {
82	+ $this->fileLocations[] = $themeStylesheet;
83	+ }
84	+ }
85	+ }

app/core/AssetManager/UIAssetMerger.php ADDED Viewed

	@@ -0,0 +1,193 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik\AssetManager;
10	+
11	+
12	+ abstract class UIAssetMerger
13	+ {
14	+ /**
15	+ * @var UIAssetFetcher
16	+ */
17	+ private $assetFetcher;
18	+
19	+ /**
20	+ * @var UIAsset
21	+ */
22	+ private $mergedAsset;
23	+
24	+ /**
25	+ * @var string
26	+ */
27	+ protected $mergedContent;
28	+
29	+ /**
30	+ * @var UIAssetCacheBuster
31	+ */
32	+ protected $cacheBuster;
33	+
34	+ /**
35	+ * @param UIAsset $mergedAsset
36	+ * @param UIAssetFetcher $assetFetcher
37	+ * @param UIAssetCacheBuster $cacheBuster
38	+ */
39	+ public function __construct($mergedAsset, $assetFetcher, $cacheBuster)
40	+ {
41	+ $this->mergedAsset = $mergedAsset;
42	+ $this->assetFetcher = $assetFetcher;
43	+ $this->cacheBuster = $cacheBuster;
44	+ }
45	+
46	+ public function generateFile()
47	+ {
48	+ if (!$this->shouldGenerate()) {
49	+ return;
50	+ }
51	+
52	+ $this->mergedContent = $this->getMergedAssets();
53	+
54	+ $this->postEvent($this->mergedContent);
55	+
56	+ $this->adjustPaths();
57	+
58	+ $this->addPreamble();
59	+
60	+ $this->writeContentToFile();
61	+ }
62	+
63	+ /**
64	+ * @return string
65	+ */
66	+ abstract protected function getMergedAssets();
67	+
68	+ /**
69	+ * @return string
70	+ */
71	+ abstract protected function generateCacheBuster();
72	+
73	+ /**
74	+ * @return string
75	+ */
76	+ abstract protected function getPreamble();
77	+
78	+ /**
79	+ * @return string
80	+ */
81	+ abstract protected function getFileSeparator();
82	+
83	+ /**
84	+ * @param UIAsset $uiAsset
85	+ * @return string
86	+ */
87	+ abstract protected function processFileContent($uiAsset);
88	+
89	+ /**
90	+ * @param string $mergedContent
91	+ */
92	+ abstract protected function postEvent(&$mergedContent);
93	+
94	+ protected function getConcatenatedAssets()
95	+ {
96	+ if (empty($this->mergedContent)) {
97	+ $this->concatenateAssets();
98	+ }
99	+
100	+ return $this->mergedContent;
101	+ }
102	+
103	+ protected function concatenateAssets()
104	+ {
105	+ $mergedContent = '';
106	+
107	+ foreach ($this->getAssetCatalog()->getAssets() as $uiAsset) {
108	+ $uiAsset->validateFile();
109	+ $content = $this->processFileContent($uiAsset);
110	+
111	+ $mergedContent .= $this->getFileSeparator() . $content;
112	+ }
113	+
114	+ $this->mergedContent = $mergedContent;
115	+ }
116	+
117	+ /**
118	+ * @return string[]
119	+ */
120	+ protected function getPlugins()
121	+ {
122	+ return $this->assetFetcher->getPlugins();
123	+ }
124	+
125	+ /**
126	+ * @return UIAssetCatalog
127	+ */
128	+ protected function getAssetCatalog()
129	+ {
130	+ return $this->assetFetcher->getCatalog();
131	+ }
132	+
133	+ /**
134	+ * @return boolean
135	+ */
136	+ private function shouldGenerate()
137	+ {
138	+ if (!$this->mergedAsset->exists()) {
139	+ return true;
140	+ }
141	+
142	+ return !$this->isFileUpToDate();
143	+ }
144	+
145	+ /**
146	+ * @return boolean
147	+ */
148	+ private function isFileUpToDate()
149	+ {
150	+ $f = fopen($this->mergedAsset->getAbsoluteLocation(), 'r');
151	+ $firstLine = fgets($f);
152	+ fclose($f);
153	+
154	+ if (!empty($firstLine) && trim($firstLine) == trim($this->getCacheBusterValue())) {
155	+ return true;
156	+ }
157	+
158	+ // Some CSS file in the merge, has changed since last merged asset was generated
159	+ // Note: we do not detect changes in @import'ed LESS files
160	+ return false;
161	+ }
162	+
163	+ private function adjustPaths()
164	+ {
165	+ $theme = $this->assetFetcher->getTheme();
166	+ // During installation theme is not yet ready
167	+ if ($theme) {
168	+ $this->mergedContent = $this->assetFetcher->getTheme()->rewriteAssetsPathToTheme($this->mergedContent);
169	+ }
170	+ }
171	+
172	+ private function writeContentToFile()
173	+ {
174	+ $this->mergedAsset->writeContent($this->mergedContent);
175	+ }
176	+
177	+ /**
178	+ * @return string
179	+ */
180	+ protected function getCacheBusterValue()
181	+ {
182	+ if (empty($this->cacheBusterValue)) {
183	+ $this->cacheBusterValue = $this->generateCacheBuster();
184	+ }
185	+
186	+ return $this->cacheBusterValue;
187	+ }
188	+
189	+ private function addPreamble()
190	+ {
191	+ $this->mergedContent = $this->getPreamble() . $this->mergedContent;
192	+ }
193	+ }

app/core/AssetManager/UIAssetMerger/JScriptUIAssetMerger.php ADDED Viewed

	@@ -0,0 +1,88 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik\AssetManager\UIAssetMerger;
10	+
11	+ use Piwik\AssetManager\UIAsset;
12	+ use Piwik\AssetManager\UIAssetCacheBuster;
13	+ use Piwik\AssetManager\UIAssetFetcher\JScriptUIAssetFetcher;
14	+ use Piwik\AssetManager\UIAssetMerger;
15	+ use Piwik\AssetManager;
16	+ use Piwik\AssetManager\UIAssetMinifier;
17	+ use Piwik\Piwik;
18	+
19	+ class JScriptUIAssetMerger extends UIAssetMerger
20	+ {
21	+ /**
22	+ * @var UIAssetMinifier
23	+ */
24	+ private $assetMinifier;
25	+
26	+ /**
27	+ * @param UIAsset $mergedAsset
28	+ * @param JScriptUIAssetFetcher $assetFetcher
29	+ * @param UIAssetCacheBuster $cacheBuster
30	+ */
31	+ public function __construct($mergedAsset, $assetFetcher, $cacheBuster)
32	+ {
33	+ parent::__construct($mergedAsset, $assetFetcher, $cacheBuster);
34	+
35	+ $this->assetMinifier = UIAssetMinifier::getInstance();
36	+ }
37	+
38	+ protected function getMergedAssets()
39	+ {
40	+ return $this->getConcatenatedAssets();
41	+ }
42	+
43	+ protected function generateCacheBuster()
44	+ {
45	+ $cacheBuster = $this->cacheBuster->piwikVersionBasedCacheBuster($this->getPlugins());
46	+ return "/* Matomo Javascript - cb=" . $cacheBuster . "*/\n";
47	+ }
48	+
49	+ protected function getPreamble()
50	+ {
51	+ return $this->getCacheBusterValue();
52	+ }
53	+
54	+ protected function postEvent(&$mergedContent)
55	+ {
56	+ $plugins = $this->getPlugins();
57	+
58	+ if (!empty($plugins)) {
59	+
60	+ /**
61	+ * Triggered after all the JavaScript files Piwik uses are minified and merged into a
62	+ * single file, but before the merged JavaScript is written to disk.
63	+ *
64	+ * Plugins can use this event to modify merged JavaScript or do something else
65	+ * with it.
66	+ *
67	+ * @param string $mergedContent The minified and merged JavaScript.
68	+ */
69	+ Piwik::postEvent('AssetManager.filterMergedJavaScripts', array(&$mergedContent), null, $plugins);
70	+ }
71	+ }
72	+
73	+ public function getFileSeparator()
74	+ {
75	+ return "\n";
76	+ }
77	+
78	+ protected function processFileContent($uiAsset)
79	+ {
80	+ $content = $uiAsset->getContent();
81	+
82	+ if (!$this->assetMinifier->isMinifiedJs($content)) {
83	+ $content = $this->assetMinifier->minifyJs($content);
84	+ }
85	+
86	+ return $content;
87	+ }
88	+ }

app/core/AssetManager/UIAssetMerger/StylesheetUIAssetMerger.php ADDED Viewed

	@@ -0,0 +1,260 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik\AssetManager\UIAssetMerger;
10	+
11	+ use Exception;
12	+ use lessc;
13	+ use Piwik\AssetManager\UIAsset;
14	+ use Piwik\AssetManager\UIAssetMerger;
15	+ use Piwik\Common;
16	+ use Piwik\Exception\StylesheetLessCompileException;
17	+ use Piwik\Piwik;
18	+ use Piwik\Plugin\Manager;
19	+
20	+ class StylesheetUIAssetMerger extends UIAssetMerger
21	+ {
22	+ /**
23	+ * @var lessc
24	+ */
25	+ private $lessCompiler;
26	+
27	+ /**
28	+ * @var UIAsset[]
29	+ */
30	+ private $cssAssetsToReplace = array();
31	+
32	+ public function __construct($mergedAsset, $assetFetcher, $cacheBuster)
33	+ {
34	+ parent::__construct($mergedAsset, $assetFetcher, $cacheBuster);
35	+
36	+ $this->lessCompiler = self::getLessCompiler();
37	+ }
38	+
39	+ protected function getMergedAssets()
40	+ {
41	+ // note: we're using setImportDir on purpose (not addImportDir)
42	+ $this->lessCompiler->setImportDir(PIWIK_DOCUMENT_ROOT);
43	+ $concatenatedAssets = $this->getConcatenatedAssets();
44	+
45	+ $this->lessCompiler->setFormatter('classic');
46	+ try {
47	+ $compiled = $this->lessCompiler->compile($concatenatedAssets);
48	+ } catch(\Exception $e) {
49	+ throw new StylesheetLessCompileException($e->getMessage());
50	+ }
51	+
52	+ foreach ($this->cssAssetsToReplace as $asset) {
53	+ // to fix #10173
54	+ $cssPath = $asset->getAbsoluteLocation();
55	+ $cssContent = $this->processFileContent($asset);
56	+ $compiled = str_replace($this->getCssStatementForReplacement($cssPath), $cssContent, $compiled);
57	+ }
58	+
59	+ $this->mergedContent = $compiled;
60	+ $this->cssAssetsToReplace = array();
61	+
62	+ return $compiled;
63	+ }
64	+
65	+ private function getCssStatementForReplacement($path)
66	+ {
67	+ return '.nonExistingSelectorOnlyForReplacementOfCssFiles { display:"' . $path . '"; }';
68	+ }
69	+
70	+ protected function concatenateAssets()
71	+ {
72	+ $concatenatedContent = '';
73	+
74	+ foreach ($this->getAssetCatalog()->getAssets() as $uiAsset) {
75	+ $uiAsset->validateFile();
76	+
77	+ try {
78	+ $path = $uiAsset->getAbsoluteLocation();
79	+ } catch (Exception $e) {
80	+ $path = null;
81	+ }
82	+
83	+ if (!empty($path) && Common::stringEndsWith($path, '.css')) {
84	+ // to fix #10173
85	+ $concatenatedContent .= "\n" . $this->getCssStatementForReplacement($path) . "\n";
86	+ $this->cssAssetsToReplace[] = $uiAsset;
87	+ } else {
88	+ $content = $this->processFileContent($uiAsset);
89	+ $concatenatedContent .= $this->getFileSeparator() . $content;
90	+ }
91	+ }
92	+
93	+ /**
94	+ * Triggered after all less stylesheets are concatenated into one long string but before it is
95	+ * minified and merged into one file.
96	+ *
97	+ * This event can be used to add less stylesheets that are not located in a file on the disc.
98	+ *
99	+ * @param string $concatenatedContent The content of all concatenated less files.
100	+ */
101	+ Piwik::postEvent('AssetManager.addStylesheets', array(&$concatenatedContent));
102	+
103	+ $this->mergedContent = $concatenatedContent;
104	+ }
105	+
106	+ /**
107	+ * @return lessc
108	+ * @throws Exception
109	+ */
110	+ private static function getLessCompiler()
111	+ {
112	+ if (!class_exists("lessc")) {
113	+ throw new Exception("Less was added to composer during 2.0. ==> Execute this command to update composer packages: \$ php composer.phar install");
114	+ }
115	+ $less = new lessc();
116	+ return $less;
117	+ }
118	+
119	+ protected function generateCacheBuster()
120	+ {
121	+ $fileHash = $this->cacheBuster->md5BasedCacheBuster($this->getConcatenatedAssets());
122	+ return "/* compile_me_once=$fileHash */";
123	+ }
124	+
125	+ protected function getPreamble()
126	+ {
127	+ return $this->getCacheBusterValue() . "\n"
128	+ . "/* Matomo CSS file is compiled with Less. You may be interested in writing a custom Theme for Matomo! */\n";
129	+ }
130	+
131	+ protected function postEvent(&$mergedContent)
132	+ {
133	+ /**
134	+ * Triggered after all less stylesheets are compiled to CSS, minified and merged into
135	+ * one file, but before the generated CSS is written to disk.
136	+ *
137	+ * This event can be used to modify merged CSS.
138	+ *
139	+ * @param string $mergedContent The merged and minified CSS.
140	+ */
141	+ Piwik::postEvent('AssetManager.filterMergedStylesheets', array(&$mergedContent));
142	+ }
143	+
144	+ public function getFileSeparator()
145	+ {
146	+ return '';
147	+ }
148	+
149	+ protected function processFileContent($uiAsset)
150	+ {
151	+ $pathsRewriter = $this->getCssPathsRewriter($uiAsset);
152	+ $content = $uiAsset->getContent();
153	+ $content = $this->rewriteCssImagePaths($content, $pathsRewriter);
154	+ $content = $this->rewriteCssImportPaths($content, $pathsRewriter);
155	+ return $content;
156	+ }
157	+
158	+ /**
159	+ * Rewrite CSS url() directives
160	+ *
161	+ * @param string $content
162	+ * @param callable $pathsRewriter
163	+ * @return string
164	+ */
165	+ private function rewriteCssImagePaths($content, $pathsRewriter)
166	+ {
167	+ $content = preg_replace_callback("/(url\(['\"]?)([^'\")]*)/", $pathsRewriter, $content);
168	+ return $content;
169	+ }
170	+
171	+ /**
172	+ * Rewrite CSS import directives
173	+ *
174	+ * @param string $content
175	+ * @param callable $pathsRewriter
176	+ * @return string
177	+ */
178	+ private function rewriteCssImportPaths($content, $pathsRewriter)
179	+ {
180	+ $content = preg_replace_callback("/(@import \")([^\")]*)/", $pathsRewriter, $content);
181	+ return $content;
182	+ }
183	+
184	+ /**
185	+ * Rewrite CSS url directives
186	+ * - rewrites paths defined relatively to their css/less definition file
187	+ * - rewrite windows directory separator \\ to /
188	+ *
189	+ * @param UIAsset $uiAsset
190	+ * @return \Closure
191	+ */
192	+ private function getCssPathsRewriter($uiAsset)
193	+ {
194	+ $baseDirectory = dirname($uiAsset->getRelativeLocation());
195	+ $webDirs = Manager::getAlternativeWebRootDirectories();
196	+
197	+ return function ($matches) use ($baseDirectory, $webDirs) {
198	+ $absolutePath = PIWIK_DOCUMENT_ROOT . "/$baseDirectory/" . $matches[2];
199	+
200	+ // Allow to import extension less file
201	+ if (strpos($matches[2], '.') === false) {
202	+ $absolutePath .= '.less';
203	+ }
204	+
205	+ // Prevent from rewriting full path
206	+ $absolutePathReal = realpath($absolutePath);
207	+ if ($absolutePathReal) {
208	+ $relativePath = $baseDirectory . "/" . $matches[2];
209	+ $relativePath = str_replace('\\', '/', $relativePath);
210	+ $publicPath = $matches[1] . $relativePath;
211	+ } else {
212	+ foreach ($webDirs as $absPath => $relativePath) {
213	+ if (strpos($baseDirectory, $relativePath) === 0) {
214	+ if (strpos($matches[2], '.') === 0) {
215	+ // eg ../images/ok.png
216	+ $fileRelative = $baseDirectory . '/' . $matches[2];
217	+ $fileAbsolute = $absPath . str_replace($relativePath, '', $fileRelative);
218	+ if (file_exists($fileAbsolute)) {
219	+ return $matches[1] . $fileRelative;
220	+ }
221	+ } elseif (strpos($matches[2], 'plugins/') === 0) {
222	+ // eg plugins/Foo/images/ok.png
223	+ $fileRelative = substr($matches[2], strlen('plugins/'));
224	+ $fileAbsolute = $absPath . $fileRelative;
225	+ if (file_exists($fileAbsolute)) {
226	+ return $matches[1] . $relativePath . $fileRelative;
227	+ }
228	+ } elseif ($matches[1] === '@import "') {
229	+ $fileRelative = $baseDirectory . '/' . $matches[2];
230	+ $fileAbsolute = $absPath . str_replace($relativePath, '', $fileRelative);
231	+ if (file_exists($fileAbsolute)) {
232	+ return $matches[1] . $baseDirectory . '/' . $matches[2];
233	+ }
234	+ }
235	+ }
236	+ }
237	+
238	+ $publicPath = $matches[1] . $matches[2];
239	+ }
240	+
241	+ return $publicPath;
242	+ };
243	+ }
244	+
245	+ /**
246	+ * @param UIAsset $uiAsset
247	+ * @return int
248	+ */
249	+ protected function countDirectoriesInPathToRoot($uiAsset)
250	+ {
251	+ $rootDirectory = realpath($uiAsset->getBaseDirectory());
252	+
253	+ if ($rootDirectory != PATH_SEPARATOR
254	+ && substr($rootDirectory, -strlen(PATH_SEPARATOR)) !== PATH_SEPARATOR) {
255	+ $rootDirectory .= PATH_SEPARATOR;
256	+ }
257	+ $rootDirectoryLen = strlen($rootDirectory);
258	+ return $rootDirectoryLen;
259	+ }
260	+ }

app/core/AssetManager/UIAssetMinifier.php ADDED Viewed

	@@ -0,0 +1,66 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ * @method static \Piwik\AssetManager\UIAssetMinifier getInstance()
9	+ */
10	+ namespace Piwik\AssetManager;
11	+
12	+ use Exception;
13	+ use JShrink\Minifier;
14	+ use Piwik\Singleton;
15	+
16	+ class UIAssetMinifier extends Singleton
17	+ {
18	+ const MINIFIED_JS_RATIO = 100;
19	+
20	+ protected function __construct()
21	+ {
22	+ self::validateDependency();
23	+ parent::__construct();
24	+ }
25	+
26	+ /**
27	+ * Indicates if the provided JavaScript content has already been minified or not.
28	+ * The heuristic is based on a custom ratio : (size of file) / (number of lines).
29	+ * The threshold (100) has been found empirically on existing files :
30	+ * - the ratio never exceeds 50 for non-minified content and
31	+ * - it never goes under 150 for minified content.
32	+ *
33	+ * @param string $content Contents of the JavaScript file
34	+ * @return boolean
35	+ */
36	+ public function isMinifiedJs($content)
37	+ {
38	+ $lineCount = substr_count($content, "\n");
39	+
40	+ if ($lineCount == 0) {
41	+ return true;
42	+ }
43	+
44	+ $contentSize = strlen($content);
45	+
46	+ $ratio = $contentSize / $lineCount;
47	+
48	+ return $ratio > self::MINIFIED_JS_RATIO;
49	+ }
50	+
51	+ /**
52	+ * @param string $content
53	+ * @return string
54	+ */
55	+ public function minifyJs($content)
56	+ {
57	+ return Minifier::minify($content);
58	+ }
59	+
60	+ private static function validateDependency()
61	+ {
62	+ if (!class_exists("JShrink\\Minifier")) {
63	+ throw new Exception("JShrink could not be found, maybe you are using Matomo from git and need to update Composer. $ php composer.phar update");
64	+ }
65	+ }
66	+ }

app/core/Auth.php ADDED Viewed

	@@ -0,0 +1,221 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+
10	+ namespace Piwik;
11	+
12	+ use Exception;
13	+
14	+ /**
15	+ * Base interface for authentication implementations.
16	+ *
17	+ * Plugins that provide Auth implementations must provide a class that implements
18	+ * this interface. Additionally, an instance of that class must be set in the
19	+ * container with the 'Piwik\Auth' key during the
20	+ * [Request.initAuthenticationObject](http://developer.piwik.org/api-reference/events#requestinitauthenticationobject)
21	+ * event.
22	+ *
23	+ * Authentication implementations must support authentication via username and
24	+ * clear-text password and authentication via username and token auth. They can
25	+ * additionally support authentication via username and an MD5 hash of a password. If
26	+ * they don't support it, then [formless authentication](http://piwik.org/faq/how-to/faq_30/) will fail.
27	+ *
28	+ * Derived implementations should favor authenticating by password over authenticating
29	+ * by token auth. That is to say, if a token auth and a password are set, password
30	+ * authentication should be used.
31	+ *
32	+ * ### Examples
33	+ *
34	+ * How an Auth implementation will be used
35	+ *
36	+ * // authenticating by password
37	+ * $auth = StaticContainer::get('Piwik\Auth');
38	+ * $auth->setLogin('user');
39	+ * $auth->setPassword('password');
40	+ * $result = $auth->authenticate();
41	+ *
42	+ * // authenticating by token auth
43	+ * $auth = StaticContainer::get('Piwik\Auth');
44	+ * $auth->setLogin('user');
45	+ * $auth->setTokenAuth('...');
46	+ * $result = $auth->authenticate();
47	+ *
48	+ * @api
49	+ */
50	+ interface Auth
51	+ {
52	+ /**
53	+ * Must return the Authentication module's name, e.g., `"Login"`.
54	+ *
55	+ * @return string
56	+ */
57	+ public function getName();
58	+
59	+ /**
60	+ * Sets the authentication token to authenticate with.
61	+ *
62	+ * @param string $token_auth authentication token
63	+ */
64	+ public function setTokenAuth($token_auth);
65	+
66	+ /**
67	+ * Returns the login of the user being authenticated.
68	+ *
69	+ * @return string
70	+ */
71	+ public function getLogin();
72	+
73	+ /**
74	+ * Returns the secret used to calculate a user's token auth.
75	+ *
76	+ * A users token auth is generated using the user's login and this secret. The secret
77	+ * should be specific to the user and not easily guessed. Piwik's default Auth implementation
78	+ * uses an MD5 hash of a user's password.
79	+ *
80	+ * @return string
81	+ * @throws Exception if the token auth secret does not exist or cannot be obtained.
82	+ */
83	+ public function getTokenAuthSecret();
84	+
85	+ /**
86	+ * Sets the login name to authenticate with.
87	+ *
88	+ * @param string $login The username.
89	+ */
90	+ public function setLogin($login);
91	+
92	+ /**
93	+ * Sets the password to authenticate with.
94	+ *
95	+ * @param string $password Password (not hashed).
96	+ */
97	+ public function setPassword($password);
98	+
99	+ /**
100	+ * Sets the hash of the password to authenticate with. The hash will be an MD5 hash.
101	+ *
102	+ * @param string $passwordHash The hashed password.
103	+ * @throws Exception if authentication by hashed password is not supported.
104	+ */
105	+ public function setPasswordHash($passwordHash);
106	+
107	+ /**
108	+ * Authenticates a user using the login and password set using the setters. Can also authenticate
109	+ * via token auth if one is set and no password is set.
110	+ *
111	+ * Note: this method must successfully authenticate if the token auth supplied is a special hash
112	+ * of the user's real token auth. This is because the SessionInitializer class stores a
113	+ * hash of the token auth in the session cookie. You can calculate the token auth hash using the
114	+ * {@link Piwik\Plugins\Login\SessionInitializer::getHashTokenAuth()} method.
115	+ *
116	+ * @return AuthResult
117	+ * @throws Exception if the Auth implementation has an invalid state (ie, no login
118	+ * was specified). Note: implementations are not required to throw
119	+ * exceptions for invalid state, but they are allowed to.
120	+ */
121	+ public function authenticate();
122	+ }
123	+
124	+ /**
125	+ * Authentication result. This is what is returned by authentication attempts using {@link Auth}
126	+ * implementations.
127	+ *
128	+ * @api
129	+ */
130	+ class AuthResult
131	+ {
132	+ const FAILURE = 0;
133	+ const SUCCESS = 1;
134	+ const SUCCESS_SUPERUSER_AUTH_CODE = 42;
135	+
136	+ /**
137	+ * token_auth parameter used to authenticate in the API
138	+ *
139	+ * @var string
140	+ */
141	+ protected $tokenAuth = null;
142	+
143	+ /**
144	+ * The login used to authenticate.
145	+ *
146	+ * @var string
147	+ */
148	+ protected $login = null;
149	+
150	+ /**
151	+ * The authentication result code. Can be self::FAILURE, self::SUCCESS, or
152	+ * self::SUCCESS_SUPERUSER_AUTH_CODE.
153	+ *
154	+ * @var int
155	+ */
156	+ protected $code = null;
157	+
158	+ /**
159	+ * Constructor for AuthResult
160	+ *
161	+ * @param int $code
162	+ * @param string $login identity
163	+ * @param string $tokenAuth
164	+ */
165	+ public function __construct($code, $login, $tokenAuth)
166	+ {
167	+ $this->code = (int)$code;
168	+ $this->login = $login;
169	+ $this->tokenAuth = $tokenAuth;
170	+ }
171	+
172	+ /**
173	+ * Returns the login used to authenticate.
174	+ *
175	+ * @return string
176	+ */
177	+ public function getIdentity()
178	+ {
179	+ return $this->login;
180	+ }
181	+
182	+ /**
183	+ * Returns the token_auth to authenticate the current user in the API
184	+ *
185	+ * @return string
186	+ */
187	+ public function getTokenAuth()
188	+ {
189	+ return $this->tokenAuth;
190	+ }
191	+
192	+ /**
193	+ * Returns the authentication result code.
194	+ *
195	+ * @return int
196	+ */
197	+ public function getCode()
198	+ {
199	+ return $this->code;
200	+ }
201	+
202	+ /**
203	+ * Returns true if the user has Super User access, false otherwise.
204	+ *
205	+ * @return bool
206	+ */
207	+ public function hasSuperUserAccess()
208	+ {
209	+ return $this->getCode() == self::SUCCESS_SUPERUSER_AUTH_CODE;
210	+ }
211	+
212	+ /**
213	+ * Returns true if this result was successfully authentication.
214	+ *
215	+ * @return bool
216	+ */
217	+ public function wasAuthenticationSuccessful()
218	+ {
219	+ return $this->code > self::FAILURE;
220	+ }
221	+ }

app/core/Auth/Password.php ADDED Viewed

	@@ -0,0 +1,69 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ */
8	+ namespace Piwik\Auth;
9	+
10	+ /**
11	+ * Main class to handle actions related to password hashing and verification.
12	+ *
13	+ * @api
14	+ */
15	+ class Password
16	+ {
17	+ /**
18	+ * Hashes a password with the configured algorithm.
19	+ *
20	+ * @param string $password
21	+ * @return string
22	+ */
23	+ public function hash($password)
24	+ {
25	+ return password_hash($password, PASSWORD_BCRYPT);
26	+ }
27	+
28	+ /**
29	+ * Returns information about a hashed password (algo, options, ...).
30	+ *
31	+ * Can be used to verify whether a string is compatible with password_hash().
32	+ *
33	+ * @param string
34	+ * @return array
35	+ */
36	+ public function info($hash)
37	+ {
38	+ return password_get_info($hash);
39	+ }
40	+
41	+ /**
42	+ * Rehashes a user's password if necessary.
43	+ *
44	+ * This method expects the password to be pre-hashed by
45	+ * \Piwik\Plugins\UsersManager\UsersManager::getPasswordHash().
46	+ *
47	+ * @param string $hash
48	+ * @return boolean
49	+ */
50	+ public function needsRehash($hash)
51	+ {
52	+ return password_needs_rehash($hash, PASSWORD_BCRYPT);
53	+ }
54	+
55	+ /**
56	+ * Verifies a user's password against the provided hash.
57	+ *
58	+ * This method expects the password to be pre-hashed by
59	+ * \Piwik\Plugins\UsersManager\UsersManager::getPasswordHash().
60	+ *
61	+ * @param string $password
62	+ * @param string $hash
63	+ * @return boolean
64	+ */
65	+ public function verify($password, $hash)
66	+ {
67	+ return password_verify($password, $hash);
68	+ }
69	+ }

app/core/BaseFactory.php ADDED Viewed

	@@ -0,0 +1,65 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik;
10	+
11	+ use Exception;
12	+
13	+ /**
14	+ * Base class for all factory types.
15	+ *
16	+ * Factory types are base classes that contain a factory method. This method is used to instantiate
17	+ * concrete instances by a specified string ID. Fatal errors do not occur if a class does not exist.
18	+ * Instead an exception is thrown.
19	+ *
20	+ * Derived classes should override the getClassNameFromClassId and getInvalidClassIdExceptionMessage
21	+ * static methods.
22	+ */
23	+ abstract class BaseFactory
24	+ {
25	+ /**
26	+ * Creates a new instance of a class using a string ID.
27	+ *
28	+ * @param string $classId The ID of the class.
29	+ * @return \Piwik\DataTable\Renderer
30	+ * @throws Exception if $classId is invalid.
31	+ */
32	+ public static function factory($classId)
33	+ {
34	+ $className = static::getClassNameFromClassId($classId);
35	+
36	+ if (!class_exists($className)) {
37	+ self::sendPlainHeader();
38	+ throw new Exception(static::getInvalidClassIdExceptionMessage($classId));
39	+ }
40	+
41	+ return new $className;
42	+ }
43	+
44	+ private static function sendPlainHeader()
45	+ {
46	+ Common::sendHeader('Content-Type: text/plain; charset=utf-8');
47	+ }
48	+
49	+ /**
50	+ * Should return a class name based on the class's associated string ID.
51	+ */
52	+ protected static function getClassNameFromClassId($id)
53	+ {
54	+ return $id;
55	+ }
56	+
57	+ /**
58	+ * Should return a message to use in an Exception when an invalid class ID is supplied to
59	+ * {@link factory()}.
60	+ */
61	+ protected static function getInvalidClassIdExceptionMessage($id)
62	+ {
63	+ return "Invalid class ID '$id' for " . get_called_class() . "::factory().";
64	+ }
65	+ }

app/core/Cache.php ADDED Viewed

	@@ -0,0 +1,117 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik;
10	+
11	+ use Piwik\Cache\Backend;
12	+ use Piwik\Container\StaticContainer;
13	+
14	+ class Cache
15	+ {
16	+
17	+ /**
18	+ * This can be considered as the default cache to use in case you don't know which one to pick. It does not support
19	+ * the caching of any objects though. Only boolean, numbers, strings and arrays are supported. Whenever you request
20	+ * an entry from the cache it will fetch the entry. Cache entries might be persisted but not necessarily. It
21	+ * depends on the configured backend.
22	+ *
23	+ * @return Cache\Lazy
24	+ */
25	+ public static function getLazyCache()
26	+ {
27	+ return StaticContainer::get('Piwik\Cache\Lazy');
28	+ }
29	+
30	+ /**
31	+ * This class is used to cache any data during one request. It won't be persisted between requests and it can
32	+ * cache all kind of data, even objects or resources. This cache is very fast.
33	+ *
34	+ * @return Cache\Transient
35	+ */
36	+ public static function getTransientCache()
37	+ {
38	+ return StaticContainer::get('Piwik\Cache\Transient');
39	+ }
40	+
41	+ /**
42	+ * This cache stores all its cache entries under one "cache" entry in a configurable backend.
43	+ *
44	+ * This comes handy for things that you need very often, nearly in every request. For example plugin metadata, the
45	+ * list of tracker plugins, the list of available languages, ...
46	+ * Instead of having to read eg. a hundred cache entries from files (or any other backend) it only loads one cache
47	+ * entry which contains the hundred keys. Should be used only for things that you need very often and only for
48	+ * cache entries that are not too large to keep loading and parsing the single cache entry fast.
49	+ * All cache entries it contains have the same life time. For fast performance it won't validate any cache ids.
50	+ * It is not possible to cache any objects using this cache.
51	+ *
52	+ * @return Cache\Eager
53	+ */
54	+ public static function getEagerCache()
55	+ {
56	+ return StaticContainer::get('Piwik\Cache\Eager');
57	+ }
58	+
59	+ public static function flushAll()
60	+ {
61	+ self::getLazyCache()->flushAll();
62	+ self::getTransientCache()->flushAll();
63	+ self::getEagerCache()->flushAll();
64	+ }
65	+
66	+ /**
67	+ * @param $type
68	+ * @return Cache\Backend
69	+ */
70	+ public static function buildBackend($type)
71	+ {
72	+ $factory = new Cache\Backend\Factory();
73	+ $options = self::getOptions($type);
74	+
75	+ $backend = $factory->buildBackend($type, $options);
76	+
77	+ return $backend;
78	+ }
79	+
80	+ private static function getOptions($type)
81	+ {
82	+ $options = self::getBackendOptions($type);
83	+
84	+ switch ($type) {
85	+ case 'file':
86	+
87	+ $options = array('directory' => StaticContainer::get('path.cache'));
88	+ break;
89	+
90	+ case 'chained':
91	+
92	+ foreach ($options['backends'] as $backend) {
93	+ $options[$backend] = self::getOptions($backend);
94	+ }
95	+
96	+ break;
97	+
98	+ case 'redis':
99	+
100	+ if (!empty($options['timeout'])) {
101	+ $options['timeout'] = (float)Common::forceDotAsSeparatorForDecimalPoint($options['timeout']);
102	+ }
103	+
104	+ break;
105	+ }
106	+
107	+ return $options;
108	+ }
109	+
110	+ private static function getBackendOptions($backend)
111	+ {
112	+ $key = ucfirst($backend) . 'Cache';
113	+ $options = Config::getInstance()->$key;
114	+
115	+ return $options;
116	+ }
117	+ }

app/core/CacheId.php ADDED Viewed

	@@ -0,0 +1,87 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik;
10	+
11	+ use Piwik\Plugin\Manager;
12	+
13	+ class CacheId
14	+ {
15	+ public static function languageAware($cacheId)
16	+ {
17	+ return $cacheId . '-' . Translate::getLanguageLoaded();
18	+ }
19	+
20	+ public static function pluginAware($cacheId)
21	+ {
22	+ $pluginManager = Manager::getInstance();
23	+ $pluginNames = $pluginManager->getLoadedPluginsName();
24	+ $cacheId = $cacheId . '-' . md5(implode('', $pluginNames));
25	+ $cacheId = self::languageAware($cacheId);
26	+
27	+ return $cacheId;
28	+ }
29	+
30	+ public static function siteAware($cacheId, array $idSites = null)
31	+ {
32	+ if ($idSites === null) {
33	+ $idSites = self::getIdSiteList('idSite');
34	+ $cacheId .= self::idSiteListCacheKey($idSites);
35	+
36	+ $idSites = self::getIdSiteList('idSites');
37	+ $cacheId .= self::idSiteListCacheKey($idSites);
38	+
39	+ $idSites = self::getIdSiteList('idsite'); // tracker param
40	+ $cacheId .= self::idSiteListCacheKey($idSites);
41	+ } else {
42	+ $cacheId .= self::idSiteListCacheKey($idSites);
43	+ }
44	+
45	+ return $cacheId;
46	+ }
47	+
48	+ private static function getIdSiteList($queryParamName)
49	+ {
50	+ if (empty($_GET[$queryParamName])
51	+ && empty($_POST[$queryParamName])
52	+ ) {
53	+ return [];
54	+ }
55	+
56	+ $idSiteGetParam = [];
57	+ if (!empty($_GET[$queryParamName])) {
58	+ $value = $_GET[$queryParamName];
59	+ $idSiteGetParam = is_array($value) ? $value : explode(',', $value);
60	+ }
61	+
62	+ $idSitePostParam = [];
63	+ if (!empty($_POST[$queryParamName])) {
64	+ $value = $_POST[$queryParamName];
65	+ $idSitePostParam = is_array($value) ? $value : explode(',', $value);
66	+ }
67	+
68	+ $idSiteList = array_merge($idSiteGetParam, $idSitePostParam);
69	+ $idSiteList = array_map('intval', $idSiteList);
70	+ $idSiteList = array_unique($idSiteList);
71	+ sort($idSiteList);
72	+ return $idSiteList;
73	+ }
74	+
75	+ private static function idSiteListCacheKey($idSites)
76	+ {
77	+ if (empty($idSites)) {
78	+ return '';
79	+ }
80	+
81	+ if (count($idSites) <= 5) {
82	+ return '-' . implode('_', $idSites); // we keep the cache key readable when possible
83	+ } else {
84	+ return '-' . md5(implode('_', $idSites)); // we need to shorten it
85	+ }
86	+ }
87	+ }

app/core/Category/Category.php ADDED Viewed

	@@ -0,0 +1,124 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ */
8	+ namespace Piwik\Category;
9	+ use Piwik\Piwik;
10	+
11	+ /**
12	+ * Base type for category. lets you change the name for a categoryId and specifiy a different order
13	+ * so the category appears eg at a different order in the reporting menu.
14	+ *
15	+ * This class is for now not exposed as public API until needed. Categories of plugins will be automatically
16	+ * displayed in the menu at the very right after all core categories.
17	+ */
18	+ class Category
19	+ {
20	+ /**
21	+ * The id of the category as specified eg in {@link Piwik\Widget\WidgetConfig::setCategoryId()`} or
22	+ * {@link Piwik\Report\getCategoryId()}. The id is used as the name in the menu and will be visible in the
23	+ * URL.
24	+ *
25	+ * @var string Should be a translation key, eg 'General_Vists'
26	+ */
27	+ protected $id = '';
28	+
29	+ /**
30	+ * @var Subcategory[]
31	+ */
32	+ protected $subcategories = array();
33	+
34	+ /**
35	+ * The order of the category. The lower the value the further left the category will appear in the menu.
36	+ * @var int
37	+ */
38	+ protected $order = 99;
39	+
40	+ /**
41	+ * The icon for this category, eg 'icon-user'
42	+ * @var int
43	+ */
44	+ protected $icon = '';
45	+
46	+ /**
47	+ * @param int $order
48	+ * @return static
49	+ */
50	+ public function setOrder($order)
51	+ {
52	+ $this->order = (int) $order;
53	+ return $this;
54	+ }
55	+
56	+ public function getOrder()
57	+ {
58	+ return $this->order;
59	+ }
60	+
61	+ public function setId($id)
62	+ {
63	+ $this->id = $id;
64	+ return $this;
65	+ }
66	+
67	+ public function getId()
68	+ {
69	+ return $this->id;
70	+ }
71	+
72	+ public function getDisplayName()
73	+ {
74	+ return Piwik::translate($this->getId());
75	+ }
76	+
77	+ public function addSubcategory(Subcategory $subcategory)
78	+ {
79	+ $subcategoryId = $subcategory->getId();
80	+
81	+ if ($this->hasSubcategory($subcategoryId)) {
82	+ throw new \Exception(sprintf('Subcategory %s already exists', $subcategoryId));
83	+ }
84	+
85	+ $this->subcategories[$subcategoryId] = $subcategory;
86	+ }
87	+
88	+ public function hasSubcategory($subcategoryId)
89	+ {
90	+ return isset($this->subcategories[$subcategoryId]);
91	+ }
92	+
93	+ public function getSubcategory($subcategoryId)
94	+ {
95	+ if ($this->hasSubcategory($subcategoryId)) {
96	+ return $this->subcategories[$subcategoryId];
97	+ }
98	+ }
99	+
100	+ /**
101	+ * @return Subcategory[]
102	+ */
103	+ public function getSubcategories()
104	+ {
105	+ return array_values($this->subcategories);
106	+ }
107	+
108	+ public function hasSubCategories()
109	+ {
110	+ return !empty($this->subcategories);
111	+ }
112	+
113	+ public function setIcon($icon)
114	+ {
115	+ $this->icon = $icon;
116	+ return $this;
117	+ }
118	+
119	+ public function getIcon()
120	+ {
121	+ return $this->icon;
122	+ }
123	+
124	+ }

app/core/Category/CategoryList.php ADDED Viewed

	@@ -0,0 +1,95 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ */
8	+ namespace Piwik\Category;
9	+
10	+ use Piwik\Container\StaticContainer;
11	+ use Piwik\Plugin;
12	+
13	+ /**
14	+ * Base type for category. lets you change the name for a categoryId and specifiy a different order
15	+ * so the category appears eg at a different order in the reporting menu.
16	+ *
17	+ * This class is for now not exposed as public API until needed. Categories of plugins will be automatically
18	+ * displayed in the menu at the very right after all core categories.
19	+ */
20	+ class CategoryList
21	+ {
22	+ /**
23	+ * @var Category[] indexed by categoryId
24	+ */
25	+ private $categories = array();
26	+
27	+ public function addCategory(Category $category)
28	+ {
29	+ $categoryId = $category->getId();
30	+
31	+ if ($this->hasCategory($categoryId)) {
32	+ throw new \Exception(sprintf('Category %s already exists', $categoryId));
33	+ }
34	+
35	+ $this->categories[$categoryId] = $category;
36	+ }
37	+
38	+ public function getCategories()
39	+ {
40	+ return $this->categories;
41	+ }
42	+
43	+ public function hasCategory($categoryId)
44	+ {
45	+ return isset($this->categories[$categoryId]);
46	+ }
47	+
48	+ /**
49	+ * Get the category having the given id, if possible.
50	+ *
51	+ * @param string $categoryId
52	+ * @return Category\|null
53	+ */
54	+ public function getCategory($categoryId)
55	+ {
56	+ if ($this->hasCategory($categoryId)) {
57	+ return $this->categories[$categoryId];
58	+ }
59	+ }
60	+
61	+ /**
62	+ * @return CategoryList
63	+ */
64	+ public static function get()
65	+ {
66	+ $list = new CategoryList();
67	+
68	+ $categories = StaticContainer::get('Piwik\Plugin\Categories');
69	+
70	+ foreach ($categories->getAllCategories() as $category) {
71	+ $list->addCategory($category);
72	+ }
73	+
74	+ // move subcategories into categories
75	+ foreach ($categories->getAllSubcategories() as $subcategory) {
76	+ $categoryId = $subcategory->getCategoryId();
77	+
78	+ if (!$categoryId) {
79	+ continue;
80	+ }
81	+
82	+ if ($list->hasCategory($categoryId)) {
83	+ $category = $list->getCategory($categoryId);
84	+ } else {
85	+ $category = new Category();
86	+ $category->setId($categoryId);
87	+ $list->addCategory($category);
88	+ }
89	+
90	+ $category->addSubcategory($subcategory);
91	+ }
92	+
93	+ return $list;
94	+ }
95	+ }

app/core/Category/Subcategory.php ADDED Viewed

	@@ -0,0 +1,146 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ */
8	+ namespace Piwik\Category;
9	+
10	+ /**
11	+ * Base type for subcategories.
12	+ *
13	+ * All widgets within a subcategory will be rendered in the Piwik reporting UI under the same page. By default
14	+ * you do not have to specify any subcategory as they are created automatically. Only create a subcategory if you
15	+ * want to change the name for a specific subcategoryId or if you want to specifiy a different order so the subcategory
16	+ * appears eg at a different order in the reporting menu. It also affects the order of reports in
17	+ * `API.getReportMetadata` and wherever we display any reports.
18	+ *
19	+ * To define a subcategory just place a subclass within the `Categories` folder of your plugin.
20	+ *
21	+ * Subcategories can also be added through the {@hook Subcategory.addSubcategories} event.
22	+ *
23	+ * @api since Piwik 3.0.0
24	+ */
25	+ class Subcategory
26	+ {
27	+ /**
28	+ * The id of the subcategory, see eg {@link Piwik\Widget\WidgetConfig::setSubcategoryId()`} or
29	+ * {@link Piwik\Report\getSubcategoryId()}. The id will be used in the Piwik reporting URL and as the name
30	+ * in the Piwik reporting submenu. If you want to define a different URL and name, specify a {@link $name}.
31	+ * For example you might want to have the actual GoalId (eg '4') in the URL but the actual goal name in the
32	+ * submenu (eg 'Downloads'). In this case one should specify `$id=4;$name='Downloads'`.
33	+ *
34	+ * @var string eg 'General_Overview' or 'VisitTime_ByServerTimeWidgetName'.
35	+ */
36	+ protected $id = '';
37	+
38	+ /**
39	+ * The id of the category the subcategory belongs to, must be specified.
40	+ * See {@link Piwik\Widget\WidgetConfig::setCategoryId()`} or {@link Piwik\Report\getCategoryId()}.
41	+ *
42	+ * @var string A translation key eg 'General_Visits' or 'Goals_Goals'
43	+ */
44	+ protected $categoryId = '';
45	+
46	+ /**
47	+ * The name that shall be used in the menu etc, defaults to the specified {@link $id}. See {@link $id}.
48	+ * @var string
49	+ */
50	+ protected $name = '';
51	+
52	+ /**
53	+ * The order of the subcategory. The lower the value the earlier a widget or a report will be displayed.
54	+ * @var int
55	+ */
56	+ protected $order = 99;
57	+
58	+ /**
59	+ * Sets (overwrites) the id of the subcategory see {@link $id}.
60	+ *
61	+ * @param string $id A translation key eg 'General_Overview'.
62	+ * @return static
63	+ */
64	+ public function setId($id)
65	+ {
66	+ $this->id = $id;
67	+ return $this;
68	+ }
69	+
70	+ /**
71	+ * Get the id of the subcategory.
72	+ * @return string
73	+ */
74	+ public function getId()
75	+ {
76	+ return $this->id;
77	+ }
78	+
79	+ /**
80	+ * Get the specifed categoryId see {@link $categoryId}.
81	+ *
82	+ * @return string
83	+ */
84	+ public function getCategoryId()
85	+ {
86	+ return $this->categoryId;
87	+ }
88	+
89	+ /**
90	+ * Sets (overwrites) the categoryId see {@link $categoryId}.
91	+ *
92	+ * @param string $categoryId
93	+ * @return static
94	+ */
95	+ public function setCategoryId($categoryId)
96	+ {
97	+ $this->categoryId = $categoryId;
98	+ return $this;
99	+ }
100	+
101	+ /**
102	+ * Sets (overwrites) the name see {@link $name} and {@link $id}.
103	+ *
104	+ * @param string $name A translation key eg 'General_Overview'.
105	+ * @return static
106	+ */
107	+ public function setName($name)
108	+ {
109	+ $this->name = $name;
110	+ return $this;
111	+ }
112	+
113	+ /**
114	+ * Get the name of the subcategory.
115	+ * @return string
116	+ */
117	+ public function getName()
118	+ {
119	+ if (!empty($this->name)) {
120	+ return $this->name;
121	+ }
122	+
123	+ return $this->id;
124	+ }
125	+
126	+ /**
127	+ * Sets (overwrites) the order see {@link $order}.
128	+ *
129	+ * @param int $order
130	+ * @return static
131	+ */
132	+ public function setOrder($order)
133	+ {
134	+ $this->order = (int) $order;
135	+ return $this;
136	+ }
137	+
138	+ /**
139	+ * Get the order of the subcategory.
140	+ * @return int
141	+ */
142	+ public function getOrder()
143	+ {
144	+ return $this->order;
145	+ }
146	+ }

app/core/CliMulti.php ADDED Viewed

	@@ -0,0 +1,471 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ */
8	+ namespace Piwik;
9	+
10	+ use Piwik\Archiver\Request;
11	+ use Piwik\CliMulti\CliPhp;
12	+ use Piwik\CliMulti\Output;
13	+ use Piwik\CliMulti\Process;
14	+ use Piwik\Container\StaticContainer;
15	+
16	+ /**
17	+ * Class CliMulti.
18	+ */
19	+ class CliMulti
20	+ {
21	+ const BASE_WAIT_TIME = 250000; // 250 * 1000 = 250ms
22	+
23	+ /**
24	+ * If set to true or false it will overwrite whether async is supported or not.
25	+ *
26	+ * @var null\|bool
27	+ */
28	+ public $supportsAsync = null;
29	+
30	+ /**
31	+ * @var Process[]
32	+ */
33	+ private $processes = array();
34	+
35	+ /**
36	+ * If set it will issue at most concurrentProcessesLimit requests
37	+ * @var int
38	+ */
39	+ private $concurrentProcessesLimit = null;
40	+
41	+ /**
42	+ * @var Output[]
43	+ */
44	+ private $outputs = array();
45	+
46	+ private $acceptInvalidSSLCertificate = false;
47	+
48	+ /**
49	+ * @var bool
50	+ */
51	+ private $runAsSuperUser = false;
52	+
53	+ /**
54	+ * Only used when doing synchronous curl requests.
55	+ *
56	+ * @var string
57	+ */
58	+ private $urlToPiwik = null;
59	+
60	+ private $phpCliOptions = '';
61	+
62	+ /**
63	+ * @var callable
64	+ */
65	+ private $onProcessFinish = null;
66	+
67	+ public function __construct()
68	+ {
69	+ $this->supportsAsync = $this->supportsAsync();
70	+ }
71	+
72	+ /**
73	+ * It will request all given URLs in parallel (async) using the CLI and wait until all requests are finished.
74	+ * If multi cli is not supported (eg windows) it will initiate an HTTP request instead (not async).
75	+ *
76	+ * @param string[] $piwikUrls An array of urls, for instance:
77	+ *
78	+ * `array('http://www.example.com/piwik?module=API...')`
79	+ *
80	+ * Make sure query parameter values are properly encoded in the URLs.
81	+ *
82	+ * @return array The response of each URL in the same order as the URLs. The array can contain null values in case
83	+ * there was a problem with a request, for instance if the process died unexpected.
84	+ */
85	+ public function request(array $piwikUrls)
86	+ {
87	+ $chunks = array($piwikUrls);
88	+ if ($this->concurrentProcessesLimit) {
89	+ $chunks = array_chunk($piwikUrls, $this->concurrentProcessesLimit);
90	+ }
91	+
92	+ $results = array();
93	+ foreach ($chunks as $urlsChunk) {
94	+ $results = array_merge($results, $this->requestUrls($urlsChunk));
95	+ }
96	+
97	+ return $results;
98	+ }
99	+
100	+ /**
101	+ * Forwards the given configuration options to the PHP cli command.
102	+ * @param string $phpCliOptions eg "-d memory_limit=8G -c=path/to/php.ini"
103	+ */
104	+ public function setPhpCliConfigurationOptions($phpCliOptions)
105	+ {
106	+ $this->phpCliOptions = (string) $phpCliOptions;
107	+ }
108	+
109	+ /**
110	+ * Ok, this sounds weird. Why should we care about ssl certificates when we are in CLI mode? It is needed for
111	+ * our simple fallback mode for Windows where we initiate HTTP requests instead of CLI.
112	+ * @param $acceptInvalidSSLCertificate
113	+ */
114	+ public function setAcceptInvalidSSLCertificate($acceptInvalidSSLCertificate)
115	+ {
116	+ $this->acceptInvalidSSLCertificate = $acceptInvalidSSLCertificate;
117	+ }
118	+
119	+ /**
120	+ * @param $limit int Maximum count of requests to issue in parallel
121	+ */
122	+ public function setConcurrentProcessesLimit($limit)
123	+ {
124	+ $this->concurrentProcessesLimit = $limit;
125	+ }
126	+
127	+ public function runAsSuperUser($runAsSuperUser = true)
128	+ {
129	+ $this->runAsSuperUser = $runAsSuperUser;
130	+ }
131	+
132	+ private function start($piwikUrls)
133	+ {
134	+ foreach ($piwikUrls as $index => $url) {
135	+ $shouldStart = null;
136	+ if ($url instanceof Request) {
137	+ $shouldStart = $url->start();
138	+ }
139	+
140	+ $cmdId = $this->generateCommandId($url) . $index;
141	+
142	+ if ($shouldStart === Request::ABORT) {
143	+ // output is needed to ensure same order of url to response
144	+ $output = new Output($cmdId);
145	+ $output->write(serialize(array('aborted' => '1')));
146	+ $this->outputs[] = $output;
147	+ } else {
148	+ $this->executeUrlCommand($cmdId, $url);
149	+ }
150	+ }
151	+ }
152	+
153	+ private function executeUrlCommand($cmdId, $url)
154	+ {
155	+ $output = new Output($cmdId);
156	+
157	+ if ($this->supportsAsync) {
158	+ $this->executeAsyncCli($url, $output, $cmdId);
159	+ } else {
160	+ $this->executeNotAsyncHttp($url, $output);
161	+ }
162	+
163	+ $this->outputs[] = $output;
164	+ }
165	+
166	+ private function buildCommand($hostname, $query, $outputFile, $doEsacpeArg = true)
167	+ {
168	+ $bin = $this->findPhpBinary();
169	+ $superuserCommand = $this->runAsSuperUser ? "--superuser" : "";
170	+
171	+ if ($doEsacpeArg) {
172	+ $hostname = escapeshellarg($hostname);
173	+ $query = escapeshellarg($query);
174	+ }
175	+
176	+ return sprintf('%s %s %s/console climulti:request -q --matomo-domain=%s %s %s > %s 2>&1 &',
177	+ $bin, $this->phpCliOptions, PIWIK_INCLUDE_PATH, $hostname, $superuserCommand, $query, $outputFile);
178	+ }
179	+
180	+ private function getResponse()
181	+ {
182	+ $response = array();
183	+
184	+ foreach ($this->outputs as $output) {
185	+ $response[] = $output->get();
186	+ }
187	+
188	+ return $response;
189	+ }
190	+
191	+ private function hasFinished()
192	+ {
193	+ foreach ($this->processes as $index => $process) {
194	+ $hasStarted = $process->hasStarted();
195	+
196	+ if (!$hasStarted && 8 <= $process->getSecondsSinceCreation()) {
197	+ // if process was created more than 8 seconds ago but still not started there must be something wrong.
198	+ // ==> declare the process as finished
199	+ $process->finishProcess();
200	+ continue;
201	+ } elseif (!$hasStarted) {
202	+ return false;
203	+ }
204	+
205	+ if ($process->isRunning()) {
206	+ return false;
207	+ }
208	+
209	+ $pid = $process->getPid();
210	+ foreach ($this->outputs as $output) {
211	+ if ($output->getOutputId() === $pid && $output->isAbnormal()) {
212	+ $process->finishProcess();
213	+ return true;
214	+ }
215	+ }
216	+
217	+ if ($process->hasFinished()) {
218	+ // prevent from checking this process over and over again
219	+ unset($this->processes[$index]);
220	+
221	+ if ($this->onProcessFinish) {
222	+ $onProcessFinish = $this->onProcessFinish;
223	+ $onProcessFinish($pid);
224	+ }
225	+ }
226	+ }
227	+
228	+ return true;
229	+ }
230	+
231	+ private function generateCommandId($command)
232	+ {
233	+ return substr(Common::hash($command . microtime(true) . rand(0, 99999)), 0, 100);
234	+ }
235	+
236	+ /**
237	+ * What is missing under windows? Detection whether a process is still running in Process::isProcessStillRunning
238	+ * and how to send a process into background in start()
239	+ */
240	+ public function supportsAsync()
241	+ {
242	+ $supportsAsync = Process::isSupported() && !Common::isPhpCgiType() && $this->findPhpBinary();
243	+
244	+ /**
245	+ * Triggered to allow plugins to force the usage of async cli multi execution or to disable it.
246	+ *
247	+ * Example
248	+ *
249	+ * public function supportsAsync(&$supportsAsync)
250	+ * {
251	+ * $supportsAsync = false; // do not allow async climulti execution
252	+ * }
253	+ *
254	+ * @param bool &$supportsAsync Whether async is supported or not.
255	+ */
256	+ Piwik::postEvent('CliMulti.supportsAsync', array(&$supportsAsync));
257	+
258	+ return $supportsAsync;
259	+ }
260	+
261	+ private function findPhpBinary()
262	+ {
263	+ $cliPhp = new CliPhp();
264	+ return $cliPhp->findPhpBinary();
265	+ }
266	+
267	+ private function cleanup()
268	+ {
269	+ foreach ($this->processes as $pid) {
270	+ $pid->finishProcess();
271	+ }
272	+
273	+ foreach ($this->outputs as $output) {
274	+ $output->destroy();
275	+ }
276	+
277	+ $this->processes = array();
278	+ $this->outputs = array();
279	+ }
280	+
281	+ /**
282	+ * Remove files older than one week. They should be cleaned up automatically after each request but for whatever
283	+ * reason there can be always some files left.
284	+ */
285	+ public static function cleanupNotRemovedFiles()
286	+ {
287	+ $timeOneWeekAgo = strtotime('-1 week');
288	+
289	+ $files = _glob(self::getTmpPath() . '/*');
290	+ if (empty($files)) {
291	+ return;
292	+ }
293	+
294	+ foreach ($files as $file) {
295	+ if (file_exists($file)) {
296	+ $timeLastModified = filemtime($file);
297	+
298	+ if ($timeLastModified !== false && $timeOneWeekAgo > $timeLastModified) {
299	+ unlink($file);
300	+ }
301	+ }
302	+ }
303	+ }
304	+
305	+ public static function getTmpPath()
306	+ {
307	+ return StaticContainer::get('path.tmp') . '/climulti';
308	+ }
309	+
310	+ public function isCommandAlreadyRunning($url)
311	+ {
312	+ if (defined('PIWIK_TEST_MODE')) {
313	+ return false; // skip check in tests as it might result in random failures
314	+ }
315	+
316	+ if (!$this->supportsAsync) {
317	+ // we cannot detect if web archive is still running
318	+ return false;
319	+ }
320	+
321	+ $query = UrlHelper::getQueryFromUrl($url, array('pid' => 'removeme'));
322	+ $hostname = Url::getHost($checkIfTrusted = false);
323	+ $commandToCheck = $this->buildCommand($hostname, $query, $output = '', $escape = false);
324	+
325	+ $currentlyRunningJobs = `ps aux`;
326	+
327	+ $posStart = strpos($commandToCheck, 'console climulti');
328	+ $posPid = strpos($commandToCheck, '&pid='); // the pid is random each time so we need to ignore it.
329	+ $shortendCommand = substr($commandToCheck, $posStart, $posPid - $posStart);
330	+ // equals eg console climulti:request -q --matomo-domain= --superuser module=API&method=API.get&idSite=1&period=month&date=2018-04-08,2018-04-30&format=php&trigger=archivephp
331	+ $shortendCommand = preg_replace("/([&])date=.*?(&\|$)/", "", $shortendCommand);
332	+ $currentlyRunningJobs = preg_replace("/([&])date=.*?(&\|$)/", "", $currentlyRunningJobs);
333	+
334	+ if (strpos($currentlyRunningJobs, $shortendCommand) !== false) {
335	+ Log::debug($shortendCommand . ' is already running');
336	+ return true;
337	+ }
338	+
339	+ return false;
340	+ }
341	+
342	+ private function executeAsyncCli($url, Output $output, $cmdId)
343	+ {
344	+ $this->processes[] = new Process($cmdId);
345	+
346	+ $url = $this->appendTestmodeParamToUrlIfNeeded($url);
347	+ $query = UrlHelper::getQueryFromUrl($url, array('pid' => $cmdId, 'runid' => getmypid()));
348	+ $hostname = Url::getHost($checkIfTrusted = false);
349	+ $command = $this->buildCommand($hostname, $query, $output->getPathToFile());
350	+
351	+ Log::debug($command);
352	+ shell_exec($command);
353	+ }
354	+
355	+ private function executeNotAsyncHttp($url, Output $output)
356	+ {
357	+ $piwikUrl = $this->urlToPiwik ?: SettingsPiwik::getPiwikUrl();
358	+ if (empty($piwikUrl)) {
359	+ $piwikUrl = 'http://' . Url::getHost() . '/';
360	+ }
361	+
362	+ $url = $piwikUrl . $url;
363	+ if (Config::getInstance()->General['force_ssl'] == 1) {
364	+ $url = str_replace("http://", "https://", $url);
365	+ }
366	+
367	+ if ($this->runAsSuperUser) {
368	+ $tokenAuths = self::getSuperUserTokenAuths();
369	+ $tokenAuth = reset($tokenAuths);
370	+
371	+ if (strpos($url, '?') === false) {
372	+ $url .= '?';
373	+ } else {
374	+ $url .= '&';
375	+ }
376	+
377	+ $url .= 'token_auth=' . $tokenAuth;
378	+ }
379	+
380	+ try {
381	+ Log::debug("Execute HTTP API request: " . $url);
382	+ $response = Http::sendHttpRequestBy('curl', $url, $timeout = 0, $userAgent = null, $destinationPath = null, $file = null, $followDepth = 0, $acceptLanguage = false, $this->acceptInvalidSSLCertificate);
383	+ $output->write($response);
384	+ } catch (\Exception $e) {
385	+ $message = "Got invalid response from API request: $url. ";
386	+
387	+ if (isset($response) && empty($response)) {
388	+ $message .= "The response was empty. This usually means a server error. This solution to this error is generally to increase the value of 'memory_limit' in your php.ini file. Please check your Web server Error Log file for more details.";
389	+ } else {
390	+ $message .= "Response was '" . $e->getMessage() . "'";
391	+ }
392	+
393	+ $output->write($message);
394	+
395	+ Log::debug($e);
396	+ }
397	+ }
398	+
399	+ private function appendTestmodeParamToUrlIfNeeded($url)
400	+ {
401	+ $isTestMode = defined('PIWIK_TEST_MODE');
402	+
403	+ if ($isTestMode && false === strpos($url, '?')) {
404	+ $url .= "?testmode=1";
405	+ } elseif ($isTestMode) {
406	+ $url .= "&testmode=1";
407	+ }
408	+
409	+ return $url;
410	+ }
411	+
412	+ /**
413	+ * @param array $piwikUrls
414	+ * @return array
415	+ */
416	+ private function requestUrls(array $piwikUrls)
417	+ {
418	+ $this->start($piwikUrls);
419	+
420	+ $startTime = time();
421	+ do {
422	+ $elapsed = time() - $startTime;
423	+ $timeToWait = $this->getTimeToWaitBeforeNextCheck($elapsed);
424	+
425	+ usleep($timeToWait);
426	+ } while (!$this->hasFinished());
427	+
428	+ $results = $this->getResponse();
429	+ $this->cleanup();
430	+
431	+ self::cleanupNotRemovedFiles();
432	+
433	+ return $results;
434	+ }
435	+
436	+ private static function getSuperUserTokenAuths()
437	+ {
438	+ $tokens = array();
439	+
440	+ /**
441	+ * Used to be in CronArchive, moved to CliMulti.
442	+ *
443	+ * @ignore
444	+ */
445	+ Piwik::postEvent('CronArchive.getTokenAuth', array(&$tokens));
446	+
447	+ return $tokens;
448	+ }
449	+
450	+ public function setUrlToPiwik($urlToPiwik)
451	+ {
452	+ $this->urlToPiwik = $urlToPiwik;
453	+ }
454	+
455	+ public function onProcessFinish(callable $callback)
456	+ {
457	+ $this->onProcessFinish = $callback;
458	+ }
459	+
460	+ // every minute that passes adds an extra 100ms to the wait time. so 5 minutes results in 500ms extra, 20mins results in 2s extra.
461	+ private function getTimeToWaitBeforeNextCheck($elapsed)
462	+ {
463	+ $minutes = floor($elapsed / 60);
464	+ return self::BASE_WAIT_TIME + $minutes * 100000; // 100 * 1000 = 100ms
465	+ }
466	+
467	+ public static function isCliMultiRequest()
468	+ {
469	+ return Common::getRequestVar('pid', false) !== false;
470	+ }
471	+ }

app/core/CliMulti/CliPhp.php ADDED Viewed

	@@ -0,0 +1,107 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ */
8	+ namespace Piwik\CliMulti;
9	+
10	+ use Piwik\Common;
11	+
12	+ class CliPhp
13	+ {
14	+
15	+ public function findPhpBinary()
16	+ {
17	+ if (defined('PHP_BINARY')) {
18	+ if ($this->isHhvmBinary(PHP_BINARY)) {
19	+ return PHP_BINARY . ' --php';
20	+ }
21	+
22	+ if ($this->isValidPhpType(PHP_BINARY)) {
23	+ return PHP_BINARY . ' -q';
24	+ }
25	+ }
26	+
27	+ $bin = '';
28	+
29	+ if (!empty($_SERVER['_']) && Common::isPhpCliMode()) {
30	+ $bin = $this->getPhpCommandIfValid($_SERVER['_']);
31	+ }
32	+
33	+ if (empty($bin) && !empty($_SERVER['argv'][0]) && Common::isPhpCliMode()) {
34	+ $bin = $this->getPhpCommandIfValid($_SERVER['argv'][0]);
35	+ }
36	+
37	+ if (empty($bin)) {
38	+ $possiblePhpPath = PHP_BINDIR . ('\\' === \DIRECTORY_SEPARATOR ? '\\php.exe' : '/php');
39	+ $bin = $this->getPhpCommandIfValid($possiblePhpPath);
40	+ }
41	+
42	+ if (!$this->isValidPhpType($bin)) {
43	+ $bin = shell_exec('which php');
44	+ }
45	+
46	+ if (!$this->isValidPhpType($bin)) {
47	+ $bin = shell_exec('which php5');
48	+ }
49	+
50	+ if (!$this->isValidPhpType($bin)) {
51	+ return false;
52	+ }
53	+
54	+ $bin = trim($bin);
55	+
56	+ if (!$this->isValidPhpVersion($bin)) {
57	+ return false;
58	+ }
59	+
60	+ $bin .= ' -q';
61	+
62	+ return $bin;
63	+ }
64	+
65	+ private function isHhvmBinary($bin)
66	+ {
67	+ return false !== strpos($bin, 'hhvm');
68	+ }
69	+
70	+ private function isValidPhpVersion($bin)
71	+ {
72	+ global $piwik_minimumPHPVersion;
73	+ $cliVersion = $this->getPhpVersion($bin);
74	+ $isCliVersionValid = version_compare($piwik_minimumPHPVersion, $cliVersion) <= 0;
75	+ return $isCliVersionValid;
76	+ }
77	+
78	+ private function isValidPhpType($path)
79	+ {
80	+ return !empty($path)
81	+ && false === strpos($path, 'fpm')
82	+ && false === strpos($path, 'cgi')
83	+ && false === strpos($path, 'phpunit')
84	+ && false === strpos($path, 'lsphp');
85	+ }
86	+
87	+ private function getPhpCommandIfValid($path)
88	+ {
89	+ if (!empty($path) && @is_executable($path)) {
90	+ if (0 === strpos($path, PHP_BINDIR) && $this->isValidPhpType($path)) {
91	+ return $path;
92	+ }
93	+ }
94	+ return null;
95	+ }
96	+
97	+ /**
98	+ * @param string $bin PHP binary
99	+ * @return string
100	+ */
101	+ private function getPhpVersion($bin)
102	+ {
103	+ $command = sprintf("%s -r 'echo phpversion();'", $bin);
104	+ $version = shell_exec($command);
105	+ return $version;
106	+ }
107	+ }

app/core/CliMulti/Output.php ADDED Viewed

	@@ -0,0 +1,76 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ */
8	+ namespace Piwik\CliMulti;
9	+
10	+ use Piwik\CliMulti;
11	+ use Piwik\Common;
12	+ use Piwik\Filesystem;
13	+
14	+ class Output
15	+ {
16	+
17	+ private $tmpFile = '';
18	+ private $outputId = null;
19	+
20	+ public function __construct($outputId)
21	+ {
22	+ if (!Filesystem::isValidFilename($outputId)) {
23	+ throw new \Exception('The given output id has an invalid format');
24	+ }
25	+
26	+ $dir = CliMulti::getTmpPath();
27	+ Filesystem::mkdir($dir);
28	+
29	+ $this->tmpFile = $dir . '/' . $outputId . '.output';
30	+ $this->outputId = $outputId;
31	+ }
32	+
33	+ public function getOutputId()
34	+ {
35	+ return $this->outputId;
36	+ }
37	+
38	+ public function write($content)
39	+ {
40	+ file_put_contents($this->tmpFile, $content);
41	+ }
42	+
43	+ public function getPathToFile()
44	+ {
45	+ return $this->tmpFile;
46	+ }
47	+
48	+ public function isAbnormal()
49	+ {
50	+ $size = Filesystem::getFileSize($this->tmpFile, 'MB');
51	+
52	+ return $size !== null && $size >= 100;
53	+ }
54	+
55	+ public function exists()
56	+ {
57	+ return file_exists($this->tmpFile);
58	+ }
59	+
60	+ public function get()
61	+ {
62	+ $content = @file_get_contents($this->tmpFile);
63	+ $search = '#!/usr/bin/env php';
64	+ if (!empty($content)
65	+ && is_string($content)
66	+ && Common::mb_substr(trim($content), 0, strlen($search)) === $search) {
67	+ $content = trim(Common::mb_substr(trim($content), strlen($search)));
68	+ }
69	+ return $content;
70	+ }
71	+
72	+ public function destroy()
73	+ {
74	+ Filesystem::deleteFileIfExists($this->tmpFile);
75	+ }
76	+ }

app/core/CliMulti/Process.php ADDED Viewed

	@@ -0,0 +1,273 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ */
8	+ namespace Piwik\CliMulti;
9	+
10	+ use Piwik\CliMulti;
11	+ use Piwik\Filesystem;
12	+ use Piwik\SettingsServer;
13	+
14	+ /**
15	+ * There are three different states
16	+ * - PID file exists with empty content: Process is created but not started
17	+ * - PID file exists with the actual process PID as content: Process is running
18	+ * - PID file does not exist: Process is marked as finished
19	+ *
20	+ * Class Process
21	+ */
22	+ class Process
23	+ {
24	+ private $pidFile = '';
25	+ private $timeCreation = null;
26	+ private $isSupported = null;
27	+ private $pid = null;
28	+
29	+ public function __construct($pid)
30	+ {
31	+ if (!Filesystem::isValidFilename($pid)) {
32	+ throw new \Exception('The given pid has an invalid format');
33	+ }
34	+
35	+ $pidDir = CliMulti::getTmpPath();
36	+ Filesystem::mkdir($pidDir);
37	+
38	+ $this->isSupported = self::isSupported();
39	+ $this->pidFile = $pidDir . '/' . $pid . '.pid';
40	+ $this->timeCreation = time();
41	+ $this->pid = $pid;
42	+
43	+ $this->markAsNotStarted();
44	+ }
45	+
46	+ public function getPid()
47	+ {
48	+ return $this->pid;
49	+ }
50	+
51	+ private function markAsNotStarted()
52	+ {
53	+ $content = $this->getPidFileContent();
54	+
55	+ if ($this->doesPidFileExist($content)) {
56	+ return;
57	+ }
58	+
59	+ $this->writePidFileContent('');
60	+ }
61	+
62	+ public function hasStarted($content = null)
63	+ {
64	+ if (is_null($content)) {
65	+ $content = $this->getPidFileContent();
66	+ }
67	+
68	+ if (!$this->doesPidFileExist($content)) {
69	+ // process is finished, this means there was a start before
70	+ return true;
71	+ }
72	+
73	+ if ('' === trim($content)) {
74	+ // pid file is overwritten by startProcess()
75	+ return false;
76	+ }
77	+
78	+ // process is probably running or pid file was not removed
79	+ return true;
80	+ }
81	+
82	+ public function hasFinished()
83	+ {
84	+ $content = $this->getPidFileContent();
85	+
86	+ return !$this->doesPidFileExist($content);
87	+ }
88	+
89	+ public function getSecondsSinceCreation()
90	+ {
91	+ return time() - $this->timeCreation;
92	+ }
93	+
94	+ public function startProcess()
95	+ {
96	+ $this->writePidFileContent(getmypid());
97	+ }
98	+
99	+ public function isRunning()
100	+ {
101	+ $content = $this->getPidFileContent();
102	+
103	+ if (!$this->doesPidFileExist($content)) {
104	+ return false;
105	+ }
106	+
107	+ if (!$this->pidFileSizeIsNormal()) {
108	+ $this->finishProcess();
109	+ return false;
110	+ }
111	+
112	+ if ($this->isProcessStillRunning($content)) {
113	+ return true;
114	+ }
115	+
116	+ if ($this->hasStarted($content)) {
117	+ $this->finishProcess();
118	+ }
119	+
120	+ return false;
121	+ }
122	+
123	+ private function pidFileSizeIsNormal()
124	+ {
125	+ $size = Filesystem::getFileSize($this->pidFile);
126	+
127	+ return $size !== null && $size < 500;
128	+ }
129	+
130	+ public function finishProcess()
131	+ {
132	+ Filesystem::deleteFileIfExists($this->pidFile);
133	+ }
134	+
135	+ private function doesPidFileExist($content)
136	+ {
137	+ return false !== $content;
138	+ }
139	+
140	+ private function isProcessStillRunning($content)
141	+ {
142	+ if (!$this->isSupported) {
143	+ return true;
144	+ }
145	+
146	+ $lockedPID = trim($content);
147	+ $runningPIDs = self::getRunningProcesses();
148	+
149	+ return !empty($lockedPID) && in_array($lockedPID, $runningPIDs);
150	+ }
151	+
152	+ private function getPidFileContent()
153	+ {
154	+ return @file_get_contents($this->pidFile);
155	+ }
156	+
157	+ private function writePidFileContent($content)
158	+ {
159	+ file_put_contents($this->pidFile, $content);
160	+ }
161	+
162	+ public static function isSupported()
163	+ {
164	+ if (SettingsServer::isWindows()) {
165	+ return false;
166	+ }
167	+
168	+ if (self::isMethodDisabled('shell_exec')) {
169	+ return false;
170	+ }
171	+
172	+ if (self::isMethodDisabled('getmypid')) {
173	+ return false;
174	+ }
175	+
176	+ if (self::isSystemNotSupported()) {
177	+ return false;
178	+ }
179	+
180	+ if (!self::commandExists('ps') \|\| !self::returnsSuccessCode('ps') \|\| !self::commandExists('awk')) {
181	+ return false;
182	+ }
183	+
184	+ if (!in_array(getmypid(), self::getRunningProcesses())) {
185	+ return false;
186	+ }
187	+
188	+ if (!self::isProcFSMounted() && !SettingsServer::isMac()) {
189	+ return false;
190	+ }
191	+
192	+ return true;
193	+ }
194	+
195	+ private static function isSystemNotSupported()
196	+ {
197	+ $uname = @shell_exec('uname -a 2> /dev/null');
198	+
199	+ if (empty($uname)) {
200	+ $uname = php_uname();
201	+ }
202	+
203	+ if (strpos($uname, 'synology') !== false) {
204	+ return true;
205	+ }
206	+ return false;
207	+ }
208	+
209	+ public static function isMethodDisabled($command)
210	+ {
211	+ if (!function_exists($command)) {
212	+ return true;
213	+ }
214	+
215	+ $disabled = explode(',', ini_get('disable_functions'));
216	+ $disabled = array_map('trim', $disabled);
217	+ return in_array($command, $disabled) \|\| !function_exists($command);
218	+ }
219	+
220	+ private static function returnsSuccessCode($command)
221	+ {
222	+ $exec = $command . ' > /dev/null 2>&1; echo $?';
223	+ $returnCode = shell_exec($exec);
224	+ $returnCode = trim($returnCode);
225	+ return 0 == (int) $returnCode;
226	+ }
227	+
228	+ private static function commandExists($command)
229	+ {
230	+ $result = @shell_exec('which ' . escapeshellarg($command) . ' 2> /dev/null');
231	+
232	+ return !empty($result);
233	+ }
234	+
235	+ /**
236	+ * ps -e requires /proc
237	+ * @return bool
238	+ */
239	+ private static function isProcFSMounted()
240	+ {
241	+ if (is_resource(@fopen('/proc', 'r'))) {
242	+ return true;
243	+ }
244	+ // Testing if /proc is a resource with @fopen fails on systems with open_basedir set.
245	+ // by using stat we not only test the existence of /proc but also confirm it's a 'proc' filesystem
246	+ $type = @shell_exec('stat -f -c "%T" /proc 2>/dev/null');
247	+ return strpos($type, 'proc') === 0;
248	+ }
249	+
250	+ public static function getListOfRunningProcesses()
251	+ {
252	+ $processes = `ps ex 2>/dev/null`;
253	+ if (empty($processes)) {
254	+ return array();
255	+ }
256	+ return explode("\n", $processes);
257	+ }
258	+
259	+ /**
260	+ * @return int[] The ids of the currently running processes
261	+ */
262	+ public static function getRunningProcesses()
263	+ {
264	+ $ids = explode("\n", trim(`ps ex 2>/dev/null \| awk '! /defunct/ {print $1}' 2>/dev/null`));
265	+
266	+ $ids = array_map('intval', $ids);
267	+ $ids = array_filter($ids, function ($id) {
268	+ return $id > 0;
269	+ });
270	+
271	+ return $ids;
272	+ }
273	+ }

app/core/CliMulti/RequestCommand.php ADDED Viewed

	@@ -0,0 +1,129 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ */
8	+
9	+ namespace Piwik\CliMulti;
10	+
11	+ use Piwik\Application\Environment;
12	+ use Piwik\Access;
13	+ use Piwik\Container\StaticContainer;
14	+ use Piwik\Db;
15	+ use Piwik\Log;
16	+ use Piwik\Option;
17	+ use Piwik\Plugin\ConsoleCommand;
18	+ use Piwik\Url;
19	+ use Piwik\UrlHelper;
20	+ use Symfony\Component\Console\Input\InputArgument;
21	+ use Symfony\Component\Console\Input\InputInterface;
22	+ use Symfony\Component\Console\Input\InputOption;
23	+ use Symfony\Component\Console\Output\OutputInterface;
24	+
25	+ /**
26	+ * RequestCommand
27	+ */
28	+ class RequestCommand extends ConsoleCommand
29	+ {
30	+ /**
31	+ * @var Environment
32	+ */
33	+ private $environment;
34	+
35	+ protected function configure()
36	+ {
37	+ $this->setName('climulti:request');
38	+ $this->setDescription('Parses and executes the given query. See Piwik\CliMulti. Intended only for system usage.');
39	+ $this->addArgument('url-query', InputArgument::REQUIRED, 'Matomo URL query string, for instance: "module=API&method=API.getPiwikVersion&token_auth=123456789"');
40	+ $this->addOption('superuser', null, InputOption::VALUE_NONE, 'If supplied, runs the code as superuser.');
41	+ }
42	+
43	+ protected function execute(InputInterface $input, OutputInterface $output)
44	+ {
45	+ $this->recreateContainerWithWebEnvironment();
46	+
47	+ $this->initHostAndQueryString($input);
48	+
49	+ if ($this->isTestModeEnabled()) {
50	+ $indexFile = '/tests/PHPUnit/proxy/';
51	+
52	+ $this->resetDatabase();
53	+ } else {
54	+ $indexFile = '/';
55	+ }
56	+
57	+ $indexFile .= 'index.php';
58	+
59	+ if (!empty($_GET['pid'])) {
60	+ $process = new Process($_GET['pid']);
61	+
62	+ if ($process->hasFinished()) {
63	+ return;
64	+ }
65	+
66	+ $process->startProcess();
67	+ }
68	+
69	+ if ($input->getOption('superuser')) {
70	+ StaticContainer::addDefinitions(array(
71	+ 'observers.global' => \DI\add(array(
72	+ array('Environment.bootstrapped', function () {
73	+ Access::getInstance()->setSuperUserAccess(true);
74	+ })
75	+ )),
76	+ ));
77	+ }
78	+
79	+ require_once PIWIK_INCLUDE_PATH . $indexFile;
80	+
81	+ if (!empty($process)) {
82	+ $process->finishProcess();
83	+ }
84	+ }
85	+
86	+ private function isTestModeEnabled()
87	+ {
88	+ return !empty($_GET['testmode']);
89	+ }
90	+
91	+ /**
92	+ * @param InputInterface $input
93	+ */
94	+ protected function initHostAndQueryString(InputInterface $input)
95	+ {
96	+ $_GET = array();
97	+
98	+ // @todo remove piwik-domain fallback in Matomo 4
99	+ $hostname = $input->getOption('matomo-domain') ?: $input->getOption('piwik-domain');
100	+ Url::setHost($hostname);
101	+
102	+ $query = $input->getArgument('url-query');
103	+ $query = UrlHelper::getArrayFromQueryString($query); // NOTE: this method can create the StaticContainer now
104	+ foreach ($query as $name => $value) {
105	+ $_GET[$name] = $value;
106	+ }
107	+ }
108	+
109	+ /**
110	+ * We will be simulating an HTTP request here (by including index.php).
111	+ *
112	+ * To avoid weird side-effects (e.g. the logging output messing up the HTTP response on the CLI output)
113	+ * we need to recreate the container with the default environment instead of the CLI environment.
114	+ */
115	+ private function recreateContainerWithWebEnvironment()
116	+ {
117	+ StaticContainer::clearContainer();
118	+ Log::unsetInstance();
119	+
120	+ $this->environment = new Environment(null);
121	+ $this->environment->init();
122	+ }
123	+
124	+ private function resetDatabase()
125	+ {
126	+ Option::clearCache();
127	+ Db::destroyDatabaseObject();
128	+ }
129	+ }

app/core/Columns/ComputedMetricFactory.php ADDED Viewed

	@@ -0,0 +1,57 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik\Columns;
10	+
11	+ use Piwik\Piwik;
12	+ use Piwik\Plugin\ArchivedMetric;
13	+ use Piwik\Plugin\ComputedMetric;
14	+ use Piwik\Plugin\Report;
15	+
16	+ /**
17	+ * A factory to create computed metrics.
18	+ *
19	+ * @api since Piwik 3.2.0
20	+ */
21	+ class ComputedMetricFactory
22	+ {
23	+ /**
24	+ * @var MetricsList
25	+ */
26	+ private $metricsList = null;
27	+
28	+ /**
29	+ * Generates a new report metric factory.
30	+ * @param MetricsList $list A report list instance
31	+ * @ignore
32	+ */
33	+ public function __construct(MetricsList $list)
34	+ {
35	+ $this->metricsList = $list;
36	+ }
37	+
38	+ /**
39	+ * @return \Piwik\Plugin\ComputedMetric
40	+ */
41	+ public function createComputedMetric($metricName1, $metricName2, $aggregation)
42	+ {
43	+ $metric1 = $this->metricsList->getMetric($metricName1);
44	+
45	+ if (!$metric1 instanceof ArchivedMetric \|\| !$metric1->getDimension()) {
46	+ throw new \Exception('Only possible to create computed metric for an archived metric with a dimension');
47	+ }
48	+
49	+ $dimension1 = $metric1->getDimension();
50	+
51	+ $metric = new ComputedMetric($metricName1, $metricName2, $aggregation);
52	+ $metric->setCategory($dimension1->getCategoryId());
53	+
54	+ return $metric;
55	+ }
56	+
57	+ }

app/core/Columns/Dimension.php ADDED Viewed

	@@ -0,0 +1,907 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik\Columns;
10	+ use Piwik\Common;
11	+ use Piwik\Db;
12	+ use Piwik\Piwik;
13	+ use Piwik\Plugin;
14	+ use Piwik\Plugin\ArchivedMetric;
15	+ use Piwik\Plugin\ComponentFactory;
16	+ use Piwik\Plugin\Segment;
17	+ use Exception;
18	+ use Piwik\CacheId;
19	+ use Piwik\Cache as PiwikCache;
20	+ use Piwik\Plugin\Manager as PluginManager;
21	+ use Piwik\Metrics\Formatter;
22	+
23	+ /**
24	+ * @api
25	+ * @since 3.1.0
26	+ */
27	+ abstract class Dimension
28	+ {
29	+ const COMPONENT_SUBNAMESPACE = 'Columns';
30	+
31	+ /**
32	+ * Segment type 'dimension'. Can be used along with {@link setType()}.
33	+ * @api
34	+ */
35	+ const TYPE_DIMENSION = 'dimension';
36	+ const TYPE_BINARY = 'binary';
37	+ const TYPE_TEXT = 'text';
38	+ const TYPE_ENUM = 'enum';
39	+ const TYPE_MONEY = 'money';
40	+ const TYPE_BYTE = 'byte';
41	+ const TYPE_DURATION_MS = 'duration_ms';
42	+ const TYPE_DURATION_S = 'duration_s';
43	+ const TYPE_NUMBER = 'number';
44	+ const TYPE_FLOAT = 'float';
45	+ const TYPE_URL = 'url';
46	+ const TYPE_DATE = 'date';
47	+ const TYPE_TIME = 'time';
48	+ const TYPE_DATETIME = 'datetime';
49	+ const TYPE_TIMESTAMP = 'timestamp';
50	+ const TYPE_BOOL = 'bool';
51	+ const TYPE_PERCENT = 'percent';
52	+
53	+ /**
54	+ * This will be the name of the column in the database table if a $columnType is specified.
55	+ * @var string
56	+ * @api
57	+ */
58	+ protected $columnName = '';
59	+
60	+ /**
61	+ * If a columnType is defined, we will create a column in the MySQL table having this type. Please make sure
62	+ * MySQL understands this type. Once you change the column type the Piwik platform will notify the user to
63	+ * perform an update which can sometimes take a long time so be careful when choosing the correct column type.
64	+ * @var string
65	+ * @api
66	+ */
67	+ protected $columnType = '';
68	+
69	+ /**
70	+ * Holds an array of segment instances
71	+ * @var Segment[]
72	+ */
73	+ protected $segments = array();
74	+
75	+ /**
76	+ * Defines what kind of data type this dimension holds. By default the type is auto-detected based on
77	+ * `$columnType` but sometimes it may be needed to correct this value. Depending on this type, a dimension will be
78	+ * formatted differently for example.
79	+ * @var string
80	+ * @api since Piwik 3.2.0
81	+ */
82	+ protected $type = '';
83	+
84	+ /**
85	+ * Translation key for name singular
86	+ * @var string
87	+ */
88	+ protected $nameSingular = '';
89	+
90	+ /**
91	+ * Translation key for name plural
92	+ * @var string
93	+ * @api since Piwik 3.2.0
94	+ */
95	+ protected $namePlural = '';
96	+
97	+ /**
98	+ * Translation key for category
99	+ * @var string
100	+ */
101	+ protected $category = '';
102	+
103	+ /**
104	+ * By defining a segment name a user will be able to filter their visitors by this column. If you do not want to
105	+ * define a segment for this dimension, simply leave the name empty.
106	+ * @api since Piwik 3.2.0
107	+ */
108	+ protected $segmentName = '';
109	+
110	+ /**
111	+ * Sets a callback which will be executed when user will call for suggested values for segment.
112	+ *
113	+ * @var callable
114	+ * @api since Piwik 3.2.0
115	+ */
116	+ protected $suggestedValuesCallback;
117	+
118	+ /**
119	+ * Here you should explain which values are accepted/useful for your segment, for example:
120	+ * "1, 2, 3, etc." or "comcast.net, proxad.net, etc.". If the value needs any special encoding you should mention
121	+ * this as well. For example "Any URL including protocol. The URL must be URL encoded."
122	+ *
123	+ * @var string
124	+ * @api since Piwik 3.2.0
125	+ */
126	+ protected $acceptValues;
127	+
128	+ /**
129	+ * Defines to which column in the MySQL database the segment belongs (if one is conifugred). Defaults to
130	+ * `$this.dbTableName . '.'. $this.columnName` but you can customize it eg like `HOUR(log_visit.visit_last_action_time)`.
131	+ *
132	+ * @param string $sqlSegment
133	+ * @api since Piwik 3.2.0
134	+ */
135	+ protected $sqlSegment;
136	+
137	+ /**
138	+ * Interesting when specifying a segment. Sometimes you want users to set segment values that differ from the way
139	+ * they are actually stored. For instance if you want to allow to filter by any URL than you might have to resolve
140	+ * this URL to an action id. Or a country name maybe has to be mapped to a 2 letter country code. You can do this by
141	+ * specifing either a callable such as `array('Classname', 'methodName')` or by passing a closure.
142	+ * There will be four values passed to the given closure or callable: `string $valueToMatch`, `string $segment`
143	+ * (see {@link setSegment()}), `string $matchType` (eg SegmentExpression::MATCH_EQUAL or any other match constant
144	+ * of this class) and `$segmentName`.
145	+ *
146	+ * If the closure returns NULL, then Piwik assumes the segment sub-string will not match any visitor.
147	+ *
148	+ * @var string\|\Closure
149	+ * @api since Piwik 3.2.0
150	+ */
151	+ protected $sqlFilter;
152	+
153	+ /**
154	+ * Similar to {@link $sqlFilter} you can map a given segment value to another value. For instance you could map
155	+ * "new" to 0, 'returning' to 1 and any other value to '2'. You can either define a callable or a closure. There
156	+ * will be only one value passed to the closure or callable which contains the value a user has set for this
157	+ * segment.
158	+ * @var string\|array
159	+ * @api since Piwik 3.2.0
160	+ */
161	+ protected $sqlFilterValue;
162	+
163	+ /**
164	+ * Defines whether this dimension (and segment based on this dimension) is available to anonymous users.
165	+ * @var bool
166	+ * @api since Piwik 3.2.0
167	+ */
168	+ protected $allowAnonymous = true;
169	+
170	+ /**
171	+ * The name of the database table this dimension refers to
172	+ * @var string
173	+ * @api
174	+ */
175	+ protected $dbTableName = '';
176	+
177	+ /**
178	+ * By default the metricId is automatically generated based on the dimensionId. This might sometimes not be as
179	+ * readable and quite long. If you want more expressive metric names like `nb_visits` compared to
180	+ * `nb_corehomevisitid`, you can eg set a metricId `visit`.
181	+ *
182	+ * @var string
183	+ * @api since Piwik 3.2.0
184	+ */
185	+ protected $metricId = '';
186	+
187	+ /**
188	+ * To be implemented when a column references another column
189	+ * @return Join\|null
190	+ * @api since Piwik 3.2.0
191	+ */
192	+ public function getDbColumnJoin()
193	+ {
194	+ return null;
195	+ }
196	+
197	+ /**
198	+ * @return Discriminator\|null
199	+ * @api since Piwik 3.2.0
200	+ */
201	+ public function getDbDiscriminator()
202	+ {
203	+ return null;
204	+ }
205	+
206	+ /**
207	+ * To be implemented when a column represents an enum.
208	+ * @return array
209	+ * @api since Piwik 3.2.0
210	+ */
211	+ public function getEnumColumnValues()
212	+ {
213	+ return array();
214	+ }
215	+
216	+ /**
217	+ * Get the metricId which is used to generate metric names based on this dimension.
218	+ * @return string
219	+ */
220	+ public function getMetricId()
221	+ {
222	+ if (!empty($this->metricId)) {
223	+ return $this->metricId;
224	+ }
225	+
226	+ $id = $this->getId();
227	+
228	+ return str_replace(array('.', ' ', '-'), '_', strtolower($id));
229	+ }
230	+
231	+ /**
232	+ * Installs the action dimension in case it is not installed yet. The installation is already implemented based on
233	+ * the {@link $columnName} and {@link $columnType}. If you want to perform additional actions beside adding the
234	+ * column to the database - for instance adding an index - you can overwrite this method. We recommend to call
235	+ * this parent method to get the minimum required actions and then add further custom actions since this makes sure
236	+ * the column will be installed correctly. We also recommend to change the default install behavior only if really
237	+ * needed. FYI: We do not directly execute those alter table statements here as we group them together with several
238	+ * other alter table statements do execute those changes in one step which results in a faster installation. The
239	+ * column will be added to the `log_link_visit_action` MySQL table.
240	+ *
241	+ * Example:
242	+ * ```
243	+ public function install()
244	+ {
245	+ $changes = parent::install();
246	+ $changes['log_link_visit_action'][] = "ADD INDEX index_idsite_servertime ( idsite, server_time )";
247	+
248	+ return $changes;
249	+ }
250	+ ```
251	+ *
252	+ * @return array An array containing the table name as key and an array of MySQL alter table statements that should
253	+ * be executed on the given table. Example:
254	+ * ```
255	+ array(
256	+ 'log_link_visit_action' => array("ADD COLUMN `$this->columnName` $this->columnType", "ADD INDEX ...")
257	+ );
258	+ ```
259	+ * @api
260	+ */
261	+ public function install()
262	+ {
263	+ if (empty($this->columnName) \|\| empty($this->columnType) \|\| empty($this->dbTableName)) {
264	+ return array();
265	+ }
266	+
267	+ // TODO if table does not exist, create it with a primary key, but at this point we cannot really create it
268	+ // cause we need to show the query in the UI first and user needs to be able to create table manually.
269	+ // we cannot return something like "create table " here as it would be returned for each table etc.
270	+ // we need to do this in column updater etc!
271	+
272	+ return array(
273	+ $this->dbTableName => array("ADD COLUMN `$this->columnName` $this->columnType")
274	+ );
275	+ }
276	+
277	+ /**
278	+ * Updates the action dimension in case the {@link $columnType} has changed. The update is already implemented based
279	+ * on the {@link $columnName} and {@link $columnType}. This method is intended not to overwritten by plugin
280	+ * developers as it is only supposed to make sure the column has the correct type. Adding additional custom "alter
281	+ * table" actions would not really work since they would be executed with every {@link $columnType} change. So
282	+ * adding an index here would be executed whenever the columnType changes resulting in an error if the index already
283	+ * exists. If an index needs to be added after the first version is released a plugin update class should be
284	+ * created since this makes sure it is only executed once.
285	+ *
286	+ * @return array An array containing the table name as key and an array of MySQL alter table statements that should
287	+ * be executed on the given table. Example:
288	+ * ```
289	+ array(
290	+ 'log_link_visit_action' => array("MODIFY COLUMN `$this->columnName` $this->columnType", "DROP COLUMN ...")
291	+ );
292	+ ```
293	+ * @ignore
294	+ */
295	+ public function update()
296	+ {
297	+ if (empty($this->columnName) \|\| empty($this->columnType) \|\| empty($this->dbTableName)) {
298	+ return array();
299	+ }
300	+
301	+ return array(
302	+ $this->dbTableName => array("MODIFY COLUMN `$this->columnName` $this->columnType")
303	+ );
304	+ }
305	+
306	+ /**
307	+ * Uninstalls the dimension if a {@link $columnName} and {@link columnType} is set. In case you perform any custom
308	+ * actions during {@link install()} - for instance adding an index - you should make sure to undo those actions by
309	+ * overwriting this method. Make sure to call this parent method to make sure the uninstallation of the column
310	+ * will be done.
311	+ * @throws Exception
312	+ * @api
313	+ */
314	+ public function uninstall()
315	+ {
316	+ if (empty($this->columnName) \|\| empty($this->columnType) \|\| empty($this->dbTableName)) {
317	+ return;
318	+ }
319	+
320	+ try {
321	+ $sql = "ALTER TABLE `" . Common::prefixTable($this->dbTableName) . "` DROP COLUMN `$this->columnName`";
322	+ Db::exec($sql);
323	+ } catch (Exception $e) {
324	+ if (!Db::get()->isErrNo($e, '1091')) {
325	+ throw $e;
326	+ }
327	+ }
328	+ }
329	+
330	+ /**
331	+ * Returns the ID of the category (typically a translation key).
332	+ * @return string
333	+ */
334	+ public function getCategoryId()
335	+ {
336	+ return $this->category;
337	+ }
338	+
339	+ /**
340	+ * Returns the translated name of this dimension which is typically in singular.
341	+ *
342	+ * @return string
343	+ */
344	+ public function getName()
345	+ {
346	+ if (!empty($this->nameSingular)) {
347	+ return Piwik::translate($this->nameSingular);
348	+ }
349	+
350	+ return $this->nameSingular;
351	+ }
352	+
353	+ /**
354	+ * Returns a translated name in plural for this dimension.
355	+ * @return string
356	+ * @api since Piwik 3.2.0
357	+ */
358	+ public function getNamePlural()
359	+ {
360	+ if (!empty($this->namePlural)) {
361	+ return Piwik::translate($this->namePlural);
362	+ }
363	+
364	+ return $this->getName();
365	+ }
366	+
367	+ /**
368	+ * Defines whether an anonymous user is allowed to view this dimension
369	+ * @return bool
370	+ * @api since Piwik 3.2.0
371	+ */
372	+ public function isAnonymousAllowed()
373	+ {
374	+ return $this->allowAnonymous;
375	+ }
376	+
377	+ /**
378	+ * Sets (overwrites) the SQL segment
379	+ * @param $segment
380	+ * @api since Piwik 3.2.0
381	+ */
382	+ public function setSqlSegment($segment)
383	+ {
384	+ $this->sqlSegment = $segment;
385	+ }
386	+
387	+ /**
388	+ * Sets (overwrites the dimension type)
389	+ * @param $type
390	+ * @api since Piwik 3.2.0
391	+ */
392	+ public function setType($type)
393	+ {
394	+ $this->type = $type;
395	+ }
396	+
397	+ /**
398	+ * A dimension should group values by using this method. Otherwise the same row may appear several times.
399	+ *
400	+ * @param mixed $value
401	+ * @param int $idSite
402	+ * @return mixed
403	+ * @api since Piwik 3.2.0
404	+ */
405	+ public function groupValue($value, $idSite)
406	+ {
407	+ switch ($this->type) {
408	+ case Dimension::TYPE_URL:
409	+ return str_replace(array('http://', 'https://'), '', $value);
410	+ case Dimension::TYPE_BOOL:
411	+ return !empty($value) ? '1' : '0';
412	+ case Dimension::TYPE_DURATION_MS:
413	+ return number_format($value / 1000, 2); // because we divide we need to group them and cannot do this in formatting step
414	+ }
415	+ return $value;
416	+ }
417	+
418	+ /**
419	+ * Formats the dimension value. By default, the dimension is formatted based on the set dimension type.
420	+ *
421	+ * @param mixed $value
422	+ * @param int $idSite
423	+ * @param Formatter $formatter
424	+ * @return mixed
425	+ * @api since Piwik 3.2.0
426	+ */
427	+ public function formatValue($value, $idSite, Formatter $formatter)
428	+ {
429	+ switch ($this->type) {
430	+ case Dimension::TYPE_BOOL:
431	+ if (empty($value)) {
432	+ return Piwik::translate('General_No');
433	+ }
434	+
435	+ return Piwik::translate('General_Yes');
436	+ case Dimension::TYPE_ENUM:
437	+ $values = $this->getEnumColumnValues();
438	+ if (isset($values[$value])) {
439	+ return $values[$value];
440	+ }
441	+ break;
442	+ case Dimension::TYPE_MONEY:
443	+ return $formatter->getPrettyMoney($value, $idSite);
444	+ case Dimension::TYPE_FLOAT:
445	+ return $formatter->getPrettyNumber((float) $value, $precision = 2);
446	+ case Dimension::TYPE_NUMBER:
447	+ return $formatter->getPrettyNumber($value);
448	+ case Dimension::TYPE_DURATION_S:
449	+ return $formatter->getPrettyTimeFromSeconds($value, $displayAsSentence = false);
450	+ case Dimension::TYPE_DURATION_MS:
451	+ return $formatter->getPrettyTimeFromSeconds($value, $displayAsSentence = true);
452	+ case Dimension::TYPE_PERCENT:
453	+ return $formatter->getPrettyPercentFromQuotient($value);
454	+ case Dimension::TYPE_BYTE:
455	+ return $formatter->getPrettySizeFromBytes($value);
456	+ }
457	+
458	+ return $value;
459	+ }
460	+
461	+ /**
462	+ * Overwrite this method to configure segments. To do so just create an instance of a {@link \Piwik\Plugin\Segment}
463	+ * class, configure it and call the {@link addSegment()} method. You can add one or more segments for this
464	+ * dimension. Example:
465	+ *
466	+ * ```
467	+ * $segment = new Segment();
468	+ * $segment->setSegment('exitPageUrl');
469	+ * $segment->setName('Actions_ColumnExitPageURL');
470	+ * $segment->setCategory('General_Visit');
471	+ * $this->addSegment($segment);
472	+ * ```
473	+ */
474	+ protected function configureSegments()
475	+ {
476	+ if ($this->segmentName && $this->category
477	+ && ($this->sqlSegment \|\| ($this->columnName && $this->dbTableName))
478	+ && $this->nameSingular) {
479	+ $segment = new Segment();
480	+ $this->addSegment($segment);
481	+ }
482	+ }
483	+
484	+ /**
485	+ * Configures metrics for this dimension.
486	+ *
487	+ * For certain dimension types, some metrics will be added automatically.
488	+ *
489	+ * @param MetricsList $metricsList
490	+ * @param DimensionMetricFactory $dimensionMetricFactory
491	+ */
492	+ public function configureMetrics(MetricsList $metricsList, DimensionMetricFactory $dimensionMetricFactory)
493	+ {
494	+ if ($this->getMetricId() && $this->dbTableName && $this->columnName && $this->getNamePlural()) {
495	+ if (in_array($this->getType(), array(self::TYPE_DATETIME, self::TYPE_DATE, self::TYPE_TIME, self::TYPE_TIMESTAMP))) {
496	+ // we do not generate any metrics from these types
497	+ return;
498	+ } elseif (in_array($this->getType(), array(self::TYPE_URL, self::TYPE_TEXT, self::TYPE_BINARY, self::TYPE_ENUM))) {
499	+ $metric = $dimensionMetricFactory->createMetric(ArchivedMetric::AGGREGATION_UNIQUE);
500	+ $metricsList->addMetric($metric);
501	+ } elseif (in_array($this->getType(), array(self::TYPE_BOOL))) {
502	+ $metric = $dimensionMetricFactory->createMetric(ArchivedMetric::AGGREGATION_SUM);
503	+ $metricsList->addMetric($metric);
504	+ } else {
505	+ $metric = $dimensionMetricFactory->createMetric(ArchivedMetric::AGGREGATION_SUM);
506	+ $metricsList->addMetric($metric);
507	+
508	+ $metric = $dimensionMetricFactory->createMetric(ArchivedMetric::AGGREGATION_MAX);
509	+ $metricsList->addMetric($metric);
510	+ }
511	+ }
512	+ }
513	+
514	+ /**
515	+ * Check whether a dimension has overwritten a specific method.
516	+ * @param $method
517	+ * @return bool
518	+ * @ignore
519	+ */
520	+ public function hasImplementedEvent($method)
521	+ {
522	+ $method = new \ReflectionMethod($this, $method);
523	+ $declaringClass = $method->getDeclaringClass();
524	+
525	+ return 0 === strpos($declaringClass->name, 'Piwik\Plugins');
526	+ }
527	+
528	+ /**
529	+ * Adds a new segment. It automatically sets the SQL segment depending on the column name in case none is set
530	+ * already.
531	+ * @see \Piwik\Columns\Dimension::addSegment()
532	+ * @param Segment $segment
533	+ * @api
534	+ */
535	+ protected function addSegment(Segment $segment)
536	+ {
537	+ if (!$segment->getSegment() && $this->segmentName) {
538	+ $segment->setSegment($this->segmentName);
539	+ }
540	+
541	+ if (!$segment->getType()) {
542	+ $metricTypes = array(self::TYPE_NUMBER, self::TYPE_FLOAT, self::TYPE_MONEY, self::TYPE_DURATION_S, self::TYPE_DURATION_MS);
543	+ if (in_array($this->getType(), $metricTypes, $strict = true)) {
544	+ $segment->setType(Segment::TYPE_METRIC);
545	+ } else {
546	+ $segment->setType(Segment::TYPE_DIMENSION);
547	+ }
548	+ }
549	+
550	+ if (!$segment->getCategoryId() && $this->category) {
551	+ $segment->setCategory($this->category);
552	+ }
553	+
554	+ if (!$segment->getName() && $this->nameSingular) {
555	+ $segment->setName($this->nameSingular);
556	+ }
557	+
558	+ $sqlSegment = $segment->getSqlSegment();
559	+
560	+ if (empty($sqlSegment) && !$segment->getUnionOfSegments()) {
561	+ if (!empty($this->sqlSegment)) {
562	+ $segment->setSqlSegment($this->sqlSegment);
563	+ } elseif ($this->dbTableName && $this->columnName) {
564	+ $segment->setSqlSegment($this->dbTableName . '.' . $this->columnName);
565	+ } else {
566	+ throw new Exception('Segment cannot be added because no sql segment is set');
567	+ }
568	+ }
569	+
570	+ if (!$this->suggestedValuesCallback) {
571	+ // we can generate effecient value callback for enums automatically
572	+ $enum = $this->getEnumColumnValues();
573	+ if (!empty($enum)) {
574	+ $this->suggestedValuesCallback = function ($idSite, $maxValuesToReturn) use ($enum) {
575	+ $values = array_values($enum);
576	+ return array_slice($values, 0, $maxValuesToReturn);
577	+ };
578	+ }
579	+ }
580	+
581	+ if (!$this->acceptValues) {
582	+ // we can generate accept values for enums automatically
583	+ $enum = $this->getEnumColumnValues();
584	+ if (!empty($enum)) {
585	+ $enumValues = array_values($enum);
586	+ $enumValues = array_slice($enumValues, 0, 20);
587	+ $this->acceptValues = 'Eg. ' . implode(', ', $enumValues);
588	+ };
589	+ }
590	+
591	+ if ($this->acceptValues && !$segment->getAcceptValues()) {
592	+ $segment->setAcceptedValues($this->acceptValues);
593	+ }
594	+
595	+ if (!$this->sqlFilterValue && !$segment->getSqlFilter() && !$segment->getSqlFilterValue()) {
596	+ // no sql filter configured, we try to configure automatically for enums
597	+ $enum = $this->getEnumColumnValues();
598	+ if (!empty($enum)) {
599	+ $this->sqlFilterValue = function ($value, $sqlSegmentName) use ($enum) {
600	+ if (isset($enum[$value])) {
601	+ return $value;
602	+ }
603	+
604	+ $id = array_search($value, $enum);
605	+
606	+ if ($id === false) {
607	+ $id = array_search(strtolower(trim(urldecode($value))), $enum);
608	+
609	+ if ($id === false) {
610	+ throw new \Exception("Invalid '$sqlSegmentName' segment value $value");
611	+ }
612	+ }
613	+
614	+ return $id;
615	+ };
616	+ };
617	+ }
618	+
619	+ if ($this->suggestedValuesCallback && !$segment->getSuggestedValuesCallback()) {
620	+ $segment->setSuggestedValuesCallback($this->suggestedValuesCallback);
621	+ }
622	+
623	+ if ($this->sqlFilterValue && !$segment->getSqlFilterValue()) {
624	+ $segment->setSqlFilterValue($this->sqlFilterValue);
625	+ }
626	+
627	+ if ($this->sqlFilter && !$segment->getSqlFilter()) {
628	+ $segment->setSqlFilter($this->sqlFilter);
629	+ }
630	+
631	+ if (!$this->allowAnonymous) {
632	+ $segment->setRequiresAtLeastViewAccess(true);
633	+ }
634	+
635	+ $this->segments[] = $segment;
636	+ }
637	+
638	+ /**
639	+ * Get the list of configured segments.
640	+ * @return Segment[]
641	+ * @ignore
642	+ */
643	+ public function getSegments()
644	+ {
645	+ if (empty($this->segments)) {
646	+ $this->configureSegments();
647	+ }
648	+
649	+ return $this->segments;
650	+ }
651	+
652	+ /**
653	+ * Returns the name of the segment that this dimension defines
654	+ * @return string
655	+ * @api since Piwik 3.2.0
656	+ */
657	+ public function getSegmentName()
658	+ {
659	+ return $this->segmentName;
660	+ }
661	+
662	+ /**
663	+ * Get the name of the dimension column.
664	+ * @return string
665	+ * @ignore
666	+ */
667	+ public function getColumnName()
668	+ {
669	+ return $this->columnName;
670	+ }
671	+
672	+ /**
673	+ * Returns a sql segment expression for this dimension.
674	+ * @return string
675	+ * @api since Piwik 3.2.0
676	+ */
677	+ public function getSqlSegment()
678	+ {
679	+ if (!empty($this->sqlSegment)) {
680	+ return $this->sqlSegment;
681	+ }
682	+
683	+ if ($this->dbTableName && $this->columnName) {
684	+ return $this->dbTableName . '.' . $this->columnName;
685	+ }
686	+ }
687	+
688	+ /**
689	+ * Check whether the dimension has a column type configured
690	+ * @return bool
691	+ * @ignore
692	+ */
693	+ public function hasColumnType()
694	+ {
695	+ return !empty($this->columnType);
696	+ }
697	+
698	+ /**
699	+ * Returns the name of the database table this dimension belongs to.
700	+ * @return string
701	+ * @api since Piwik 3.2.0
702	+ */
703	+ public function getDbTableName()
704	+ {
705	+ return $this->dbTableName;
706	+ }
707	+
708	+ /**
709	+ * Returns a unique string ID for this dimension. The ID is built using the namespaced class name
710	+ * of the dimension, but is modified to be more human readable.
711	+ *
712	+ * @return string eg, `"Referrers.Keywords"`
713	+ * @throws Exception if the plugin and simple class name of this instance cannot be determined.
714	+ * This would only happen if the dimension is located in the wrong directory.
715	+ * @api
716	+ */
717	+ public function getId()
718	+ {
719	+ $className = get_class($this);
720	+
721	+ return $this->generateIdFromClass($className);
722	+ }
723	+
724	+ /**
725	+ * @param string $className
726	+ * @return string
727	+ * @throws Exception
728	+ * @ignore
729	+ */
730	+ protected function generateIdFromClass($className)
731	+ {
732	+ // parse plugin name & dimension name
733	+ $regex = "/Piwik\\\\Plugins\\\\([^\\\\]+)\\\\" . self::COMPONENT_SUBNAMESPACE . "\\\\([^\\\\]+)/";
734	+ if (!preg_match($regex, $className, $matches)) {
735	+ throw new Exception("'$className' is located in the wrong directory.");
736	+ }
737	+
738	+ $pluginName = $matches[1];
739	+ $dimensionName = $matches[2];
740	+
741	+ return $pluginName . '.' . $dimensionName;
742	+ }
743	+
744	+ /**
745	+ * Gets an instance of all available visit, action and conversion dimension.
746	+ * @return Dimension[]
747	+ */
748	+ public static function getAllDimensions()
749	+ {
750	+ $cacheId = CacheId::siteAware(CacheId::pluginAware('AllDimensions'));
751	+ $cache = PiwikCache::getTransientCache();
752	+
753	+ if (!$cache->contains($cacheId)) {
754	+ $plugins = PluginManager::getInstance()->getPluginsLoadedAndActivated();
755	+ $instances = array();
756	+
757	+ /**
758	+ * Triggered to add new dimensions that cannot be picked up automatically by the platform.
759	+ * This is useful if the plugin allows a user to create reports / dimensions dynamically. For example
760	+ * CustomDimensions or CustomVariables. There are a variable number of dimensions in this case and it
761	+ * wouldn't be really possible to create a report file for one of these dimensions as it is not known
762	+ * how many Custom Dimensions will exist.
763	+ *
764	+ * Example
765	+ *
766	+ * public function addDimension(&$dimensions)
767	+ * {
768	+ * $dimensions[] = new MyCustomDimension();
769	+ * }
770	+ *
771	+ * @param Dimension[] $reports An array of dimensions
772	+ */
773	+ Piwik::postEvent('Dimension.addDimensions', array(&$instances));
774	+
775	+ foreach ($plugins as $plugin) {
776	+ foreach (self::getDimensions($plugin) as $instance) {
777	+ $instances[] = $instance;
778	+ }
779	+ }
780	+
781	+ /**
782	+ * Triggered to filter / restrict dimensions.
783	+ *
784	+ * Example
785	+ *
786	+ * public function filterDimensions(&$dimensions)
787	+ * {
788	+ * foreach ($dimensions as $index => $dimension) {
789	+ * if ($dimension->getName() === 'Page URL') {}
790	+ * unset($dimensions[$index]); // remove this dimension
791	+ * }
792	+ * }
793	+ * }
794	+ *
795	+ * @param Dimension[] $dimensions An array of dimensions
796	+ */
797	+ Piwik::postEvent('Dimension.filterDimensions', array(&$instances));
798	+
799	+ $cache->save($cacheId, $instances);
800	+ }
801	+
802	+ return $cache->fetch($cacheId);
803	+ }
804	+
805	+ public static function getDimensions(Plugin $plugin)
806	+ {
807	+ $columns = $plugin->findMultipleComponents('Columns', '\\Piwik\\Columns\\Dimension');
808	+ $instances = array();
809	+
810	+ foreach ($columns as $colum) {
811	+ $instances[] = new $colum();
812	+ }
813	+
814	+ return $instances;
815	+ }
816	+
817	+ /**
818	+ * Creates a Dimension instance from a string ID (see {@link getId()}).
819	+ *
820	+ * @param string $dimensionId See {@link getId()}.
821	+ * @return Dimension\|null The created instance or null if there is no Dimension for
822	+ * $dimensionId or if the plugin that contains the Dimension is
823	+ * not loaded.
824	+ * @api
825	+ * @deprecated Please use DimensionsProvider::factory instead
826	+ */
827	+ public static function factory($dimensionId)
828	+ {
829	+ list($module, $dimension) = explode('.', $dimensionId);
830	+ return ComponentFactory::factory($module, $dimension, __CLASS__);
831	+ }
832	+
833	+ /**
834	+ * Returns the name of the plugin that contains this Dimension.
835	+ *
836	+ * @return string
837	+ * @throws Exception if the Dimension is not located within a Plugin module.
838	+ * @api
839	+ */
840	+ public function getModule()
841	+ {
842	+ $id = $this->getId();
843	+ if (empty($id)) {
844	+ throw new Exception("Invalid dimension ID: '$id'.");
845	+ }
846	+
847	+ $parts = explode('.', $id);
848	+ return reset($parts);
849	+ }
850	+
851	+ /**
852	+ * Returns the type of the dimension which defines what kind of value this dimension stores.
853	+ * @return string
854	+ * @api since Piwik 3.2.0
855	+ */
856	+ public function getType()
857	+ {
858	+ if (!empty($this->type)) {
859	+ return $this->type;
860	+ }
861	+
862	+ if ($this->getDbColumnJoin()) {
863	+ // best guess
864	+ return self::TYPE_TEXT;
865	+ }
866	+
867	+ if ($this->getEnumColumnValues()) {
868	+ // best guess
869	+ return self::TYPE_ENUM;
870	+ }
871	+
872	+ if (!empty($this->columnType)) {
873	+ // best guess
874	+ $type = strtolower($this->columnType);
875	+ if (strpos($type, 'datetime') !== false) {
876	+ return self::TYPE_DATETIME;
877	+ } elseif (strpos($type, 'timestamp') !== false) {
878	+ return self::TYPE_TIMESTAMP;
879	+ } elseif (strpos($type, 'date') !== false) {
880	+ return self::TYPE_DATE;
881	+ } elseif (strpos($type, 'time') !== false) {
882	+ return self::TYPE_TIME;
883	+ } elseif (strpos($type, 'float') !== false) {
884	+ return self::TYPE_FLOAT;
885	+ } elseif (strpos($type, 'decimal') !== false) {
886	+ return self::TYPE_FLOAT;
887	+ } elseif (strpos($type, 'int') !== false) {
888	+ return self::TYPE_NUMBER;
889	+ } elseif (strpos($type, 'binary') !== false) {
890	+ return self::TYPE_BINARY;
891	+ }
892	+ }
893	+
894	+ return self::TYPE_TEXT;
895	+ }
896	+
897	+ /**
898	+ * Get the version of the dimension which is used for update checks.
899	+ * @return string
900	+ * @ignore
901	+ */
902	+ public function getVersion()
903	+ {
904	+ return $this->columnType;
905	+ }
906	+
907	+ }

app/core/Columns/DimensionMetricFactory.php ADDED Viewed

	@@ -0,0 +1,122 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik\Columns;
10	+
11	+ use Piwik\Piwik;
12	+ use Piwik\Plugin\ArchivedMetric;
13	+ use Piwik\Plugin\ComputedMetric;
14	+ use Piwik\Plugin\Report;
15	+
16	+
17	+ /**
18	+ * A factory to create metrics from a dimension.
19	+ *
20	+ * @api since Piwik 3.2.0
21	+ */
22	+ class DimensionMetricFactory
23	+ {
24	+ /**
25	+ * @var Dimension
26	+ */
27	+ private $dimension = null;
28	+
29	+ /**
30	+ * Generates a new dimension metric factory.
31	+ * @param Dimension $dimension A dimension instance the created metrics should be based on.
32	+ */
33	+ public function __construct(Dimension $dimension)
34	+ {
35	+ $this->dimension = $dimension;
36	+ }
37	+
38	+ /**
39	+ * @return ArchivedMetric
40	+ */
41	+ public function createCustomMetric($metricName, $readableName, $aggregation, $documentation = '')
42	+ {
43	+ if (!$this->dimension->getDbTableName() \|\| !$this->dimension->getColumnName()) {
44	+ throw new \Exception(sprintf('Cannot make metric from dimension %s because DB table or column missing', $this->dimension->getId()));
45	+ }
46	+
47	+ $metric = new ArchivedMetric($this->dimension, $aggregation);
48	+ $metric->setType($this->dimension->getType());
49	+ $metric->setName($metricName);
50	+ $metric->setTranslatedName($readableName);
51	+ $metric->setDocumentation($documentation);
52	+ $metric->setCategory($this->dimension->getCategoryId());
53	+
54	+ return $metric;
55	+ }
56	+
57	+ /**
58	+ * @return \Piwik\Plugin\ComputedMetric
59	+ */
60	+ public function createComputedMetric($metricName1, $metricName2, $aggregation)
61	+ {
62	+ // We cannot use reuse ComputedMetricFactory here as it would result in an endless loop since ComputedMetricFactory
63	+ // requires a MetricsList which is just being built here...
64	+ $metric = new ComputedMetric($metricName1, $metricName2, $aggregation);
65	+ $metric->setCategory($this->dimension->getCategoryId());
66	+ return $metric;
67	+ }
68	+
69	+ /**
70	+ * @return ArchivedMetric
71	+ */
72	+ public function createMetric($aggregation)
73	+ {
74	+ $dimension = $this->dimension;
75	+
76	+ if (!$dimension->getNamePlural()) {
77	+ throw new \Exception(sprintf('No metric can be created for this dimension %s automatically because no $namePlural is set.', $dimension->getId()));
78	+ }
79	+
80	+ $prefix = '';
81	+ $translatedName = $dimension->getNamePlural();
82	+
83	+ $documentation = '';
84	+
85	+ switch ($aggregation) {
86	+ case ArchivedMetric::AGGREGATION_COUNT;
87	+ $prefix = ArchivedMetric::AGGREGATION_COUNT_PREFIX;
88	+ $translatedName = $dimension->getNamePlural();
89	+ $documentation = Piwik::translate('General_ComputedMetricCountDocumentation', $dimension->getNamePlural());
90	+ break;
91	+ case ArchivedMetric::AGGREGATION_SUM;
92	+ $prefix = ArchivedMetric::AGGREGATION_SUM_PREFIX;
93	+ $translatedName = Piwik::translate('General_ComputedMetricSum', $dimension->getNamePlural());
94	+ $documentation = Piwik::translate('General_ComputedMetricSumDocumentation', $dimension->getNamePlural());
95	+ break;
96	+ case ArchivedMetric::AGGREGATION_MAX;
97	+ $prefix = ArchivedMetric::AGGREGATION_MAX_PREFIX;
98	+ $translatedName = Piwik::translate('General_ComputedMetricMax', $dimension->getNamePlural());
99	+ $documentation = Piwik::translate('General_ComputedMetricMaxDocumentation', $dimension->getNamePlural());
100	+ break;
101	+ case ArchivedMetric::AGGREGATION_MIN;
102	+ $prefix = ArchivedMetric::AGGREGATION_MIN_PREFIX;
103	+ $translatedName = Piwik::translate('General_ComputedMetricMin', $dimension->getNamePlural());
104	+ $documentation = Piwik::translate('General_ComputedMetricMinDocumentation', $dimension->getNamePlural());
105	+ break;
106	+ case ArchivedMetric::AGGREGATION_UNIQUE;
107	+ $prefix = ArchivedMetric::AGGREGATION_UNIQUE_PREFIX;
108	+ $translatedName = Piwik::translate('General_ComputedMetricUniqueCount', $dimension->getNamePlural());
109	+ $documentation = Piwik::translate('General_ComputedMetricUniqueCountDocumentation', $dimension->getNamePlural());
110	+ break;
111	+ case ArchivedMetric::AGGREGATION_COUNT_WITH_NUMERIC_VALUE;
112	+ $prefix = ArchivedMetric::AGGREGATION_COUNT_WITH_NUMERIC_VALUE_PREFIX;
113	+ $translatedName = Piwik::translate('General_ComputedMetricCountWithValue', $dimension->getName());
114	+ $documentation = Piwik::translate('General_ComputedMetricCountWithValueDocumentation', $dimension->getName());
115	+ break;
116	+ }
117	+
118	+ $metricId = strtolower($dimension->getMetricId());
119	+
120	+ return $this->createCustomMetric($prefix . $metricId, $translatedName, $aggregation, $documentation);
121	+ }
122	+ }

app/core/Columns/DimensionsProvider.php ADDED Viewed

	@@ -0,0 +1,62 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik\Columns;
10	+
11	+ use Piwik\CacheId;
12	+ use Piwik\Cache as PiwikCache;
13	+
14	+ class DimensionsProvider
15	+ {
16	+ /**
17	+ * @param $dimensionId
18	+ * @return Dimension
19	+ */
20	+ public function factory($dimensionId)
21	+ {
22	+ $listDimensions = self::getMapOfNameToDimension();
23	+
24	+ if (empty($listDimensions) \|\| !is_array($listDimensions) \|\| !$dimensionId \|\| !array_key_exists($dimensionId, $listDimensions)) {
25	+ return null;
26	+ }
27	+
28	+ return $listDimensions[$dimensionId];
29	+ }
30	+
31	+ private static function getMapOfNameToDimension()
32	+ {
33	+ $cacheId = CacheId::siteAware(CacheId::pluginAware('DimensionFactoryMap'));
34	+
35	+ $cache = PiwikCache::getTransientCache();
36	+ if ($cache->contains($cacheId)) {
37	+ $mapIdToDimension = $cache->fetch($cacheId);
38	+ } else {
39	+ $dimensions = new static();
40	+ $dimensions = $dimensions->getAllDimensions();
41	+
42	+ $mapIdToDimension = array();
43	+ foreach ($dimensions as $dimension) {
44	+ $mapIdToDimension[$dimension->getId()] = $dimension;
45	+ }
46	+
47	+ $cache->save($cacheId, $mapIdToDimension);
48	+ }
49	+
50	+ return $mapIdToDimension;
51	+ }
52	+
53	+ /**
54	+ * Returns a list of all available dimensions.
55	+ * @return Dimension[]
56	+ */
57	+ public function getAllDimensions()
58	+ {
59	+ return Dimension::getAllDimensions();
60	+ }
61	+
62	+ }

app/core/Columns/Discriminator.php ADDED Viewed

	@@ -0,0 +1,75 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik\Columns;
10	+
11	+ use Exception;
12	+ use Piwik\Plugins\Actions\Actions\ActionSiteSearch;
13	+
14	+ /**
15	+ * @api
16	+ * @since 3.1.0
17	+ */
18	+ class Discriminator
19	+ {
20	+ private $table;
21	+ private $discriminatorColumn;
22	+ private $discriminatorValue;
23	+
24	+ /**
25	+ * Join constructor.
26	+ * @param string $table unprefixed table name
27	+ * @param null\|string $discriminatorColumn
28	+ * @param null\|int $discriminatorValue should be only hard coded, safe values.
29	+ * @throws Exception
30	+ */
31	+ public function __construct($table, $discriminatorColumn = null, $discriminatorValue = null)
32	+ {
33	+ if (empty($discriminatorColumn) \|\| !isset($discriminatorValue)) {
34	+ throw new Exception('Both discriminatorColumn and discriminatorValue need to be defined');
35	+ }
36	+ $this->table = $table;
37	+ $this->discriminatorColumn = $discriminatorColumn;
38	+ $this->discriminatorValue = $discriminatorValue;
39	+
40	+ if (!$this->isValid()) {
41	+ // if adding another string value please post an event instead to get a list of allowed values
42	+ throw new Exception('$discriminatorValue needs to be null or numeric');
43	+ }
44	+ }
45	+
46	+ public function isValid()
47	+ {
48	+ return isset($this->discriminatorColumn)
49	+ && (is_numeric($this->discriminatorValue) \|\| $this->discriminatorValue == ActionSiteSearch::CVAR_KEY_SEARCH_CATEGORY);
50	+ }
51	+
52	+ /**
53	+ * @return string
54	+ */
55	+ public function getTable()
56	+ {
57	+ return $this->table;
58	+ }
59	+
60	+ /**
61	+ * @return string
62	+ */
63	+ public function getColumn()
64	+ {
65	+ return $this->discriminatorColumn;
66	+ }
67	+
68	+ /**
69	+ * @return int\|null
70	+ */
71	+ public function getValue()
72	+ {
73	+ return $this->discriminatorValue;
74	+ }
75	+ }

app/core/Columns/Join.php ADDED Viewed

	@@ -0,0 +1,61 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik\Columns;
10	+
11	+ use Exception;
12	+
13	+ /**
14	+ * @api
15	+ * @since 3.1.0
16	+ */
17	+ class Join
18	+ {
19	+ private $table;
20	+ private $column;
21	+ private $targetColumn;
22	+
23	+ /**
24	+ * Join constructor.
25	+ * @param $table
26	+ * @param $column
27	+ * @param $targetColumn
28	+ * @throws Exception
29	+ */
30	+ public function __construct($table, $column, $targetColumn)
31	+ {
32	+ $this->table = $table;
33	+ $this->column = $column;
34	+ $this->targetColumn = $targetColumn;
35	+ }
36	+
37	+ /**
38	+ * @return string
39	+ */
40	+ public function getTable()
41	+ {
42	+ return $this->table;
43	+ }
44	+
45	+ /**
46	+ * @return string
47	+ */
48	+ public function getColumn()
49	+ {
50	+ return $this->column;
51	+ }
52	+
53	+ /**
54	+ * @return string
55	+ */
56	+ public function getTargetColumn()
57	+ {
58	+ return $this->targetColumn;
59	+ }
60	+
61	+ }

app/core/Columns/Join/ActionNameJoin.php ADDED Viewed

	@@ -0,0 +1,24 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik\Columns\Join;
10	+
11	+ use Piwik\Columns;
12	+
13	+ /**
14	+ * @api
15	+ * @since 3.1.0
16	+ */
17	+ class ActionNameJoin extends Columns\Join
18	+ {
19	+ public function __construct()
20	+ {
21	+ return parent::__construct('log_action', 'idaction', 'name');
22	+ }
23	+
24	+ }

app/core/Columns/Join/GoalNameJoin.php ADDED Viewed

	@@ -0,0 +1,24 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik\Columns\Join;
10	+
11	+ use Piwik\Columns;
12	+
13	+ /**
14	+ * @api
15	+ * @since 3.1.0
16	+ */
17	+ class GoalNameJoin extends Columns\Join
18	+ {
19	+ public function __construct()
20	+ {
21	+ return parent::__construct('goal', 'idgoal', 'name');
22	+ }
23	+
24	+ }

app/core/Columns/Join/SiteNameJoin.php ADDED Viewed

	@@ -0,0 +1,24 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik\Columns\Join;
10	+
11	+ use Piwik\Columns;
12	+
13	+ /**
14	+ * @api
15	+ * @since 3.1.0
16	+ */
17	+ class SiteNameJoin extends Columns\Join
18	+ {
19	+ public function __construct()
20	+ {
21	+ return parent::__construct('site', 'idsite', 'name');
22	+ }
23	+
24	+ }

app/core/Columns/MetricsList.php ADDED Viewed

	@@ -0,0 +1,191 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik\Columns;
10	+
11	+ use Piwik\Cache;
12	+ use Piwik\CacheId;
13	+ use Piwik\Piwik;
14	+ use Piwik\Plugin\ArchivedMetric;
15	+ use Piwik\Plugin\Metric;
16	+ use Piwik\Plugin\ProcessedMetric;
17	+
18	+ /**
19	+ * Manages the global list of metrics that can be used in reports.
20	+ *
21	+ * Metrics are added automatically by dimensions as well as through the {@hook Metric.addMetrics} and
22	+ * {@hook Metric.addComputedMetrics} and filtered through the {@hook Metric.filterMetrics} event.
23	+ * Observers for this event should call the {@link addMetric()} method to add metrics or use any of the other
24	+ * methods to remove metrics.
25	+ *
26	+ * @api since Piwik 3.2.0
27	+ */
28	+ class MetricsList
29	+ {
30	+ /**
31	+ * List of metrics
32	+ *
33	+ * @var Metric[]
34	+ */
35	+ private $metrics = array();
36	+
37	+ private $metricsByNameCache = array();
38	+
39	+ /**
40	+ * @param Metric $metric
41	+ */
42	+ public function addMetric(Metric $metric)
43	+ {
44	+ $this->metrics[] = $metric;
45	+ $this->metricsByNameCache = array();
46	+ }
47	+
48	+ /**
49	+ * Get all available metrics.
50	+ *
51	+ * @return Metric[]
52	+ */
53	+ public function getMetrics()
54	+ {
55	+ return $this->metrics;
56	+ }
57	+
58	+ /**
59	+ * Removes one or more metrics from the metrics list.
60	+ *
61	+ * @param string $metricCategory The metric category id. Can be a translation token eg 'General_Visits'
62	+ * see {@link Metric::getCategory()}.
63	+ * @param string\|false $metricName The name of the metric to remove eg 'nb_visits'.
64	+ * If not supplied, all metrics within that category will be removed.
65	+ */
66	+ public function remove($metricCategory, $metricName = false)
67	+ {
68	+ foreach ($this->metrics as $index => $metric) {
69	+ if ($metric->getCategoryId() === $metricCategory) {
70	+ if (!$metricName \|\| $metric->getName() === $metricName) {
71	+ unset($this->metrics[$index]);
72	+ $this->metricsByNameCache = array();
73	+ }
74	+ }
75	+ }
76	+ }
77	+
78	+ /**
79	+ * @param string $metricName
80	+ * @return Metric\|ArchivedMetric\|null
81	+ */
82	+ public function getMetric($metricName)
83	+ {
84	+ if (empty($this->metricsByNameCache)) {
85	+ // this method might be called quite often... eg when having heaps of goals... need to cache it
86	+ foreach ($this->metrics as $index => $metric) {
87	+ $this->metricsByNameCache[$metric->getName()] = $metric;
88	+ }
89	+ }
90	+
91	+ if (!empty($this->metricsByNameCache[$metricName])) {
92	+ return $this->metricsByNameCache[$metricName];
93	+ }
94	+
95	+ return null;
96	+ }
97	+
98	+ /**
99	+ * Get all metrics defined in the Piwik platform.
100	+ * @ignore
101	+ * @return static
102	+ */
103	+ public static function get()
104	+ {
105	+ $cache = Cache::getTransientCache();
106	+ $cacheKey = CacheId::siteAware('MetricsList');
107	+
108	+ if ($cache->contains($cacheKey)) {
109	+ return $cache->fetch($cacheKey);
110	+ }
111	+
112	+ $list = new static;
113	+
114	+ /**
115	+ * Triggered to add new metrics that cannot be picked up automatically by the platform.
116	+ * This is useful if the plugin allows a user to create metrics dynamically. For example
117	+ * CustomDimensions or CustomVariables.
118	+ *
119	+ * Example
120	+ *
121	+ * public function addMetric(&$list)
122	+ * {
123	+ * $list->addMetric(new MyCustomMetric());
124	+ * }
125	+ *
126	+ * @param MetricsList $list An instance of the MetricsList. You can add metrics to the list this way.
127	+ */
128	+ Piwik::postEvent('Metric.addMetrics', array($list));
129	+
130	+ $dimensions = Dimension::getAllDimensions();
131	+ foreach ($dimensions as $dimension) {
132	+ $factory = new DimensionMetricFactory($dimension);
133	+ $dimension->configureMetrics($list, $factory);
134	+ }
135	+
136	+ $computedFactory = new ComputedMetricFactory($list);
137	+
138	+ /**
139	+ * Triggered to add new metrics that cannot be picked up automatically by the platform.
140	+ * This is useful if the plugin allows a user to create metrics dynamically. For example
141	+ * CustomDimensions or CustomVariables.
142	+ *
143	+ * Example
144	+ *
145	+ * public function addMetric(&$list)
146	+ * {
147	+ * $list->addMetric(new MyCustomMetric());
148	+ * }
149	+ *
150	+ * @param MetricsList $list An instance of the MetricsList. You can add metrics to the list this way.
151	+ */
152	+ Piwik::postEvent('Metric.addComputedMetrics', array($list, $computedFactory));
153	+
154	+ /**
155	+ * Triggered to filter metrics.
156	+ *
157	+ * Example
158	+ *
159	+ * public function removeMetrics(Piwik\Columns\MetricsList $list)
160	+ * {
161	+ * $list->remove($category='General_Visits'); // remove all metrics having this category
162	+ * }
163	+ *
164	+ * @param MetricsList $list An instance of the MetricsList. You can change the list of metrics this way.
165	+ */
166	+ Piwik::postEvent('Metric.filterMetrics', array($list));
167	+
168	+ $availableMetrics = array();
169	+ foreach ($list->getMetrics() as $metric) {
170	+ $availableMetrics[] = $metric->getName();
171	+ }
172	+
173	+ foreach ($list->metrics as $index => $metric) {
174	+ if ($metric instanceof ProcessedMetric) {
175	+ $depMetrics = $metric->getDependentMetrics();
176	+ if (is_array($depMetrics)) {
177	+ foreach ($depMetrics as $depMetric) {
178	+ if (!in_array($depMetric, $availableMetrics, $strict = true)) {
179	+ unset($list->metrics[$index]); // not resolvable metric
180	+ }
181	+ }
182	+ }
183	+ }
184	+ }
185	+
186	+ $cache->save($cacheKey, $list);
187	+
188	+ return $list;
189	+ }
190	+
191	+ }

app/core/Columns/Updater.php ADDED Viewed

	@@ -0,0 +1,380 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik\Columns;
10	+
11	+ use Piwik\Common;
12	+ use Piwik\DbHelper;
13	+ use Piwik\Plugin\Dimension\ActionDimension;
14	+ use Piwik\Plugin\Dimension\VisitDimension;
15	+ use Piwik\Plugin\Dimension\ConversionDimension;
16	+ use Piwik\Db;
17	+ use Piwik\Plugin\Manager;
18	+ use Piwik\Updater as PiwikUpdater;
19	+ use Piwik\Filesystem;
20	+ use Piwik\Cache as PiwikCache;
21	+ use Piwik\Updater\Migration;
22	+
23	+ /**
24	+ * Class that handles dimension updates
25	+ */
26	+ class Updater extends \Piwik\Updates
27	+ {
28	+ private static $cacheId = 'AllDimensionModifyTime';
29	+
30	+ /**
31	+ * @var VisitDimension[]
32	+ */
33	+ public $visitDimensions;
34	+
35	+ /**
36	+ * @var ActionDimension[]
37	+ */
38	+ private $actionDimensions;
39	+
40	+ /**
41	+ * @var ConversionDimension[]
42	+ */
43	+ private $conversionDimensions;
44	+
45	+ /**
46	+ * @param VisitDimension[]\|null $visitDimensions
47	+ * @param ActionDimension[]\|null $actionDimensions
48	+ * @param ConversionDimension[]\|null $conversionDimensions
49	+ */
50	+ public function __construct(array $visitDimensions = null, array $actionDimensions = null, array $conversionDimensions = null)
51	+ {
52	+ $this->visitDimensions = $visitDimensions;
53	+ $this->actionDimensions = $actionDimensions;
54	+ $this->conversionDimensions = $conversionDimensions;
55	+ }
56	+
57	+ /**
58	+ * @param PiwikUpdater $updater
59	+ * @return Migration\Db[]
60	+ */
61	+ public function getMigrationQueries(PiwikUpdater $updater)
62	+ {
63	+ $sqls = array();
64	+
65	+ $changingColumns = $this->getUpdates($updater);
66	+ $errorCodes = array(
67	+ Migration\Db\Sql::ERROR_CODE_COLUMN_NOT_EXISTS,
68	+ Migration\Db\Sql::ERROR_CODE_DUPLICATE_COLUMN
69	+ );
70	+
71	+ foreach ($changingColumns as $table => $columns) {
72	+ if (empty($columns) \|\| !is_array($columns)) {
73	+ continue;
74	+ }
75	+
76	+ $sql = "ALTER TABLE `" . Common::prefixTable($table) . "` " . implode(', ', $columns);
77	+ $sqls[] = new Migration\Db\Sql($sql, $errorCodes);
78	+ }
79	+
80	+ return $sqls;
81	+ }
82	+
83	+ public function doUpdate(PiwikUpdater $updater)
84	+ {
85	+ $updater->executeMigrationQueries(__FILE__, $this->getMigrationQueries($updater));
86	+ }
87	+
88	+ private function getVisitDimensions()
89	+ {
90	+ // see eg https://github.com/piwik/piwik/issues/8399 we fetch them only on demand to improve performance
91	+ if (!isset($this->visitDimensions)) {
92	+ $this->visitDimensions = VisitDimension::getAllDimensions();
93	+ }
94	+
95	+ return $this->visitDimensions;
96	+ }
97	+
98	+ private function getActionDimensions()
99	+ {
100	+ // see eg https://github.com/piwik/piwik/issues/8399 we fetch them only on demand to improve performance
101	+ if (!isset($this->actionDimensions)) {
102	+ $this->actionDimensions = ActionDimension::getAllDimensions();
103	+ }
104	+
105	+ return $this->actionDimensions;
106	+ }
107	+
108	+ private function getConversionDimensions()
109	+ {
110	+ // see eg https://github.com/piwik/piwik/issues/8399 we fetch them only on demand to improve performance
111	+ if (!isset($this->conversionDimensions)) {
112	+ $this->conversionDimensions = ConversionDimension::getAllDimensions();
113	+ }
114	+
115	+ return $this->conversionDimensions;
116	+ }
117	+
118	+ private function getUpdates(PiwikUpdater $updater)
119	+ {
120	+ $visitColumns = DbHelper::getTableColumns(Common::prefixTable('log_visit'));
121	+ $actionColumns = DbHelper::getTableColumns(Common::prefixTable('log_link_visit_action'));
122	+ $conversionColumns = DbHelper::getTableColumns(Common::prefixTable('log_conversion'));
123	+
124	+ $allUpdatesToRun = array();
125	+
126	+ foreach ($this->getVisitDimensions() as $dimension) {
127	+ $updates = $this->getUpdatesForDimension($updater, $dimension, 'log_visit.', $visitColumns);
128	+ $allUpdatesToRun = $this->mixinUpdates($allUpdatesToRun, $updates);
129	+ }
130	+
131	+ foreach ($this->getActionDimensions() as $dimension) {
132	+ $updates = $this->getUpdatesForDimension($updater, $dimension, 'log_link_visit_action.', $actionColumns);
133	+ $allUpdatesToRun = $this->mixinUpdates($allUpdatesToRun, $updates);
134	+ }
135	+
136	+ foreach ($this->getConversionDimensions() as $dimension) {
137	+ $updates = $this->getUpdatesForDimension($updater, $dimension, 'log_conversion.', $conversionColumns);
138	+ $allUpdatesToRun = $this->mixinUpdates($allUpdatesToRun, $updates);
139	+ }
140	+
141	+ return $allUpdatesToRun;
142	+ }
143	+
144	+ /**
145	+ * @param ActionDimension\|ConversionDimension\|VisitDimension $dimension
146	+ * @param string $componentPrefix
147	+ * @return array
148	+ */
149	+ private function getUpdatesForDimension(PiwikUpdater $updater, $dimension, $componentPrefix, $existingColumnsInDb)
150	+ {
151	+ $column = $dimension->getColumnName();
152	+ $componentName = $componentPrefix . $column;
153	+
154	+ if (!$updater->hasNewVersion($componentName)) {
155	+ return array();
156	+ }
157	+
158	+ if (array_key_exists($column, $existingColumnsInDb)) {
159	+ $sqlUpdates = $dimension->update();
160	+ } else {
161	+ $sqlUpdates = $dimension->install();
162	+ }
163	+
164	+ return $sqlUpdates;
165	+ }
166	+
167	+ private function mixinUpdates($allUpdatesToRun, $updatesFromDimension)
168	+ {
169	+ if (!empty($updatesFromDimension)) {
170	+ foreach ($updatesFromDimension as $table => $col) {
171	+ if (empty($allUpdatesToRun[$table])) {
172	+ $allUpdatesToRun[$table] = $col;
173	+ } else {
174	+ $allUpdatesToRun[$table] = array_merge($allUpdatesToRun[$table], $col);
175	+ }
176	+ }
177	+ }
178	+
179	+ return $allUpdatesToRun;
180	+ }
181	+
182	+ public function getAllVersions(PiwikUpdater $updater)
183	+ {
184	+ // to avoid having to load all dimensions on each request we check if there were any changes on the file system
185	+ // can easily save > 100ms for each request
186	+ $cachedTimes = self::getCachedDimensionFileChanges();
187	+ $currentTimes = self::getCurrentDimensionFileChanges();
188	+ $diff = array_diff_assoc($currentTimes, $cachedTimes);
189	+
190	+ if (empty($diff)) {
191	+ return array();
192	+ }
193	+
194	+ $versions = array();
195	+
196	+ $visitColumns = DbHelper::getTableColumns(Common::prefixTable('log_visit'));
197	+ $actionColumns = DbHelper::getTableColumns(Common::prefixTable('log_link_visit_action'));
198	+ $conversionColumns = DbHelper::getTableColumns(Common::prefixTable('log_conversion'));
199	+
200	+ foreach ($this->getVisitDimensions() as $dimension) {
201	+ $versions = $this->mixinVersions($updater, $dimension, VisitDimension::INSTALLER_PREFIX, $visitColumns, $versions);
202	+ }
203	+
204	+ foreach ($this->getActionDimensions() as $dimension) {
205	+ $versions = $this->mixinVersions($updater, $dimension, ActionDimension::INSTALLER_PREFIX, $actionColumns, $versions);
206	+ }
207	+
208	+ foreach ($this->getConversionDimensions() as $dimension) {
209	+ $versions = $this->mixinVersions($updater, $dimension, ConversionDimension::INSTALLER_PREFIX, $conversionColumns, $versions);
210	+ }
211	+
212	+ return $versions;
213	+ }
214	+
215	+ /**
216	+ * @param PiwikUpdater $updater
217	+ * @param Dimension $dimension
218	+ * @param string $componentPrefix
219	+ * @param array $columns
220	+ * @param array $versions
221	+ * @return array The modified versions array
222	+ */
223	+ private function mixinVersions(PiwikUpdater $updater, $dimension, $componentPrefix, $columns, $versions)
224	+ {
225	+ $columnName = $dimension->getColumnName();
226	+
227	+ // dimensions w/o columns do not need DB updates
228	+ if (!$columnName \|\| !$dimension->hasColumnType()) {
229	+ return $versions;
230	+ }
231	+
232	+ $component = $componentPrefix . $columnName;
233	+ $version = $dimension->getVersion();
234	+
235	+ // if the column exists in the table, but has no associated version, and was one of the core columns
236	+ // that was moved when the dimension refactor took place, then:
237	+ // - set the installed version in the DB to the current code version
238	+ // - and do not check for updates since we just set the version to the latest
239	+ if (array_key_exists($columnName, $columns)
240	+ && false === $updater->getCurrentComponentVersion($component)
241	+ && self::wasDimensionMovedFromCoreToPlugin($component, $version)
242	+ ) {
243	+ $updater->markComponentSuccessfullyUpdated($component, $version);
244	+ return $versions;
245	+ }
246	+
247	+ $versions[$component] = $version;
248	+
249	+ return $versions;
250	+ }
251	+
252	+ public static function isDimensionComponent($name)
253	+ {
254	+ return 0 === strpos($name, 'log_visit.')
255	+ \|\| 0 === strpos($name, 'log_conversion.')
256	+ \|\| 0 === strpos($name, 'log_conversion_item.')
257	+ \|\| 0 === strpos($name, 'log_link_visit_action.');
258	+ }
259	+
260	+ public static function wasDimensionMovedFromCoreToPlugin($name, $version)
261	+ {
262	+ // maps names of core dimension columns that were part of the original dimension refactor with their
263	+ // initial "version" strings. The '1' that is sometimes appended to the end of the string (sometimes seen as
264	+ // NULL1) is from individual dimension "versioning" logic (eg, see VisitDimension::getVersion())
265	+ $initialCoreDimensionVersions = array(
266	+ 'log_visit.config_resolution' => 'VARCHAR(9) NOT NULL',
267	+ 'log_visit.config_device_brand' => 'VARCHAR( 100 ) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL',
268	+ 'log_visit.config_device_model' => 'VARCHAR( 100 ) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL',
269	+ 'log_visit.config_windowsmedia' => 'TINYINT(1) NOT NULL',
270	+ 'log_visit.config_silverlight' => 'TINYINT(1) NOT NULL',
271	+ 'log_visit.config_java' => 'TINYINT(1) NOT NULL',
272	+ 'log_visit.config_gears' => 'TINYINT(1) NOT NULL',
273	+ 'log_visit.config_pdf' => 'TINYINT(1) NOT NULL',
274	+ 'log_visit.config_quicktime' => 'TINYINT(1) NOT NULL',
275	+ 'log_visit.config_realplayer' => 'TINYINT(1) NOT NULL',
276	+ 'log_visit.config_device_type' => 'TINYINT( 100 ) NULL DEFAULT NULL',
277	+ 'log_visit.visitor_localtime' => 'TIME NOT NULL',
278	+ 'log_visit.location_region' => 'char(2) DEFAULT NULL1',
279	+ 'log_visit.visitor_days_since_last' => 'SMALLINT(5) UNSIGNED NOT NULL',
280	+ 'log_visit.location_longitude' => 'float(10, 6) DEFAULT NULL1',
281	+ 'log_visit.visit_total_events' => 'SMALLINT(5) UNSIGNED NOT NULL',
282	+ 'log_visit.config_os_version' => 'VARCHAR( 100 ) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL',
283	+ 'log_visit.location_city' => 'varchar(255) DEFAULT NULL1',
284	+ 'log_visit.location_country' => 'CHAR(3) NOT NULL1',
285	+ 'log_visit.location_latitude' => 'float(10, 6) DEFAULT NULL1',
286	+ 'log_visit.config_flash' => 'TINYINT(1) NOT NULL',
287	+ 'log_visit.config_director' => 'TINYINT(1) NOT NULL',
288	+ 'log_visit.visit_total_time' => 'SMALLINT(5) UNSIGNED NOT NULL',
289	+ 'log_visit.visitor_count_visits' => 'SMALLINT(5) UNSIGNED NOT NULL1',
290	+ 'log_visit.visit_entry_idaction_name' => 'INTEGER(11) UNSIGNED NOT NULL',
291	+ 'log_visit.visit_entry_idaction_url' => 'INTEGER(11) UNSIGNED NOT NULL',
292	+ 'log_visit.visitor_returning' => 'TINYINT(1) NOT NULL1',
293	+ 'log_visit.visitor_days_since_order' => 'SMALLINT(5) UNSIGNED NOT NULL1',
294	+ 'log_visit.visit_goal_buyer' => 'TINYINT(1) NOT NULL',
295	+ 'log_visit.visit_first_action_time' => 'DATETIME NOT NULL',
296	+ 'log_visit.visit_goal_converted' => 'TINYINT(1) NOT NULL',
297	+ 'log_visit.visitor_days_since_first' => 'SMALLINT(5) UNSIGNED NOT NULL1',
298	+ 'log_visit.visit_exit_idaction_name' => 'INTEGER(11) UNSIGNED NOT NULL',
299	+ 'log_visit.visit_exit_idaction_url' => 'INTEGER(11) UNSIGNED NULL DEFAULT 0',
300	+ 'log_visit.config_browser_version' => 'VARCHAR(20) NOT NULL',
301	+ 'log_visit.config_browser_name' => 'VARCHAR(10) NOT NULL',
302	+ 'log_visit.config_browser_engine' => 'VARCHAR(10) NOT NULL',
303	+ 'log_visit.location_browser_lang' => 'VARCHAR(20) NOT NULL',
304	+ 'log_visit.config_os' => 'CHAR(3) NOT NULL',
305	+ 'log_visit.config_cookie' => 'TINYINT(1) NOT NULL',
306	+ 'log_visit.referer_url' => 'TEXT NOT NULL',
307	+ 'log_visit.visit_total_searches' => 'SMALLINT(5) UNSIGNED NOT NULL',
308	+ 'log_visit.visit_total_actions' => 'SMALLINT(5) UNSIGNED NOT NULL',
309	+ 'log_visit.referer_keyword' => 'VARCHAR(255) NULL1',
310	+ 'log_visit.referer_name' => 'VARCHAR(70) NULL1',
311	+ 'log_visit.referer_type' => 'TINYINT(1) UNSIGNED NULL1',
312	+ 'log_visit.user_id' => 'VARCHAR(200) NULL',
313	+ 'log_link_visit_action.idaction_name' => 'INTEGER(10) UNSIGNED',
314	+ 'log_link_visit_action.idaction_url' => 'INTEGER(10) UNSIGNED DEFAULT NULL',
315	+ 'log_link_visit_action.server_time' => 'DATETIME NOT NULL',
316	+ 'log_link_visit_action.time_spent_ref_action' => 'INTEGER(10) UNSIGNED NOT NULL',
317	+ 'log_link_visit_action.idaction_event_action' => 'INTEGER(10) UNSIGNED DEFAULT NULL',
318	+ 'log_link_visit_action.idaction_event_category' => 'INTEGER(10) UNSIGNED DEFAULT NULL',
319	+ 'log_conversion.revenue_discount' => 'float default NULL',
320	+ 'log_conversion.revenue' => 'float default NULL',
321	+ 'log_conversion.revenue_shipping' => 'float default NULL',
322	+ 'log_conversion.revenue_subtotal' => 'float default NULL',
323	+ 'log_conversion.revenue_tax' => 'float default NULL',
324	+ );
325	+
326	+ if (!array_key_exists($name, $initialCoreDimensionVersions)) {
327	+ return false;
328	+ }
329	+
330	+ return strtolower($initialCoreDimensionVersions[$name]) === strtolower($version);
331	+ }
332	+
333	+ public function onNoUpdateAvailable($versionsThatWereChecked)
334	+ {
335	+ if (!empty($versionsThatWereChecked)) {
336	+ // invalidate cache only if there were actually file changes before, otherwise we write the cache on each
337	+ // request. There were versions checked only if there was a file change but no update, meaning we can
338	+ // set the cache and declare this state as "no update available".
339	+ self::cacheCurrentDimensionFileChanges();
340	+ }
341	+ }
342	+
343	+ private static function getCurrentDimensionFileChanges()
344	+ {
345	+ $times = array();
346	+ foreach (Manager::getPluginsDirectories() as $pluginsDir) {
347	+ $files = Filesystem::globr($pluginsDir . '/Columns', '.php');
348	+
349	+ foreach ($files as $file) {
350	+ $times[$file] = filemtime($file);
351	+ }
352	+ }
353	+
354	+ return $times;
355	+ }
356	+
357	+ private static function cacheCurrentDimensionFileChanges()
358	+ {
359	+ $changes = self::getCurrentDimensionFileChanges();
360	+
361	+ $cache = self::buildCache();
362	+ $cache->save(self::$cacheId, $changes);
363	+ }
364	+
365	+ private static function buildCache()
366	+ {
367	+ return PiwikCache::getEagerCache();
368	+ }
369	+
370	+ private static function getCachedDimensionFileChanges()
371	+ {
372	+ $cache = self::buildCache();
373	+
374	+ if ($cache->contains(self::$cacheId)) {
375	+ return $cache->fetch(self::$cacheId);
376	+ }
377	+
378	+ return array();
379	+ }
380	+ }

app/core/Common.php ADDED Viewed

	@@ -0,0 +1,1331 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik;
10	+
11	+ use Exception;
12	+ use Piwik\CliMulti\Process;
13	+ use Piwik\Container\StaticContainer;
14	+ use Piwik\Intl\Data\Provider\LanguageDataProvider;
15	+ use Piwik\Intl\Data\Provider\RegionDataProvider;
16	+ use Piwik\Plugins\UserCountry\LocationProvider\DefaultProvider;
17	+ use Piwik\Tracker\Cache as TrackerCache;
18	+
19	+ /**
20	+ * Contains helper methods used by both Piwik Core and the Piwik Tracking engine.
21	+ *
22	+ * This is the only non-Tracker class loaded by the \/piwik.php file.
23	+ */
24	+ class Common
25	+ {
26	+ // constants used to map the referrer type to an integer in the log_visit table
27	+ const REFERRER_TYPE_DIRECT_ENTRY = 1;
28	+ const REFERRER_TYPE_SEARCH_ENGINE = 2;
29	+ const REFERRER_TYPE_WEBSITE = 3;
30	+ const REFERRER_TYPE_CAMPAIGN = 6;
31	+ const REFERRER_TYPE_SOCIAL_NETWORK = 7;
32	+
33	+ // Flag used with htmlspecialchar. See php.net/htmlspecialchars.
34	+ const HTML_ENCODING_QUOTE_STYLE = ENT_QUOTES;
35	+
36	+ public static $isCliMode = null;
37	+
38	+ /*
39	+ * Database
40	+ */
41	+ const LANGUAGE_CODE_INVALID = 'xx';
42	+
43	+ /**
44	+ * Hashes a string into an integer which should be very low collision risks
45	+ * @param string $string String to hash
46	+ * @return int Resulting int hash
47	+ */
48	+ public static function hashStringToInt($string)
49	+ {
50	+ $stringHash = substr(md5($string), 0, 8);
51	+ return base_convert($stringHash, 16, 10);
52	+ }
53	+
54	+ /**
55	+ * Returns a prefixed table name.
56	+ *
57	+ * The table prefix is determined by the `[database] tables_prefix` INI config
58	+ * option.
59	+ *
60	+ * @param string $table The table name to prefix, ie "log_visit"
61	+ * @return string The prefixed name, ie "piwik-production_log_visit".
62	+ * @api
63	+ */
64	+ public static function prefixTable($table)
65	+ {
66	+ $prefix = Config::getInstance()->database['tables_prefix'];
67	+ return $prefix . $table;
68	+ }
69	+
70	+ /**
71	+ * Returns an array containing the prefixed table names of every passed argument.
72	+ *
73	+ * @param string ... The table names to prefix, ie "log_visit"
74	+ * @return array The prefixed names in an array.
75	+ */
76	+ public static function prefixTables()
77	+ {
78	+ $result = array();
79	+ foreach (func_get_args() as $table) {
80	+ $result[] = self::prefixTable($table);
81	+ }
82	+ return $result;
83	+ }
84	+
85	+ /**
86	+ * Removes the prefix from a table name and returns the result.
87	+ *
88	+ * The table prefix is determined by the `[database] tables_prefix` INI config
89	+ * option.
90	+ *
91	+ * @param string $table The prefixed table name, eg "piwik-production_log_visit".
92	+ * @return string The unprefixed table name, eg "log_visit".
93	+ * @api
94	+ */
95	+ public static function unprefixTable($table)
96	+ {
97	+ static $prefixTable = null;
98	+ if (is_null($prefixTable)) {
99	+ $prefixTable = Config::getInstance()->database['tables_prefix'];
100	+ }
101	+ if (empty($prefixTable)
102	+ \|\| strpos($table, $prefixTable) !== 0
103	+ ) {
104	+ return $table;
105	+ }
106	+ $count = 1;
107	+ return str_replace($prefixTable, '', $table, $count);
108	+ }
109	+
110	+ /*
111	+ * Tracker
112	+ */
113	+ public static function isGoalPluginEnabled()
114	+ {
115	+ return Plugin\Manager::getInstance()->isPluginActivated('Goals');
116	+ }
117	+
118	+ public static function isActionsPluginEnabled()
119	+ {
120	+ return Plugin\Manager::getInstance()->isPluginActivated('Actions');
121	+ }
122	+
123	+ /**
124	+ * Returns true if PHP was invoked from command-line interface (shell)
125	+ *
126	+ * @since added in 0.4.4
127	+ * @return bool true if PHP invoked as a CGI or from CLI
128	+ */
129	+ public static function isPhpCliMode()
130	+ {
131	+ if (is_bool(self::$isCliMode)) {
132	+ return self::$isCliMode;
133	+ }
134	+
135	+ if(PHP_SAPI == 'cli'){
136	+ return true;
137	+ }
138	+
139	+ if(self::isPhpCgiType() && (!isset($_SERVER['REMOTE_ADDR']) \|\| empty($_SERVER['REMOTE_ADDR']))){
140	+ return true;
141	+ }
142	+
143	+ return false;
144	+ }
145	+
146	+ /**
147	+ * Returns true if PHP is executed as CGI type.
148	+ *
149	+ * @since added in 0.4.4
150	+ * @return bool true if PHP invoked as a CGI
151	+ */
152	+ public static function isPhpCgiType()
153	+ {
154	+ $sapiType = php_sapi_name();
155	+
156	+ return substr($sapiType, 0, 3) === 'cgi';
157	+ }
158	+
159	+ /**
160	+ * Returns true if the current request is a console command, eg.
161	+ * ./console xx:yy
162	+ * or
163	+ * php console xx:yy
164	+ *
165	+ * @return bool
166	+ */
167	+ public static function isRunningConsoleCommand()
168	+ {
169	+ $searched = 'console';
170	+ $consolePos = strpos($_SERVER['SCRIPT_NAME'], $searched);
171	+ $expectedConsolePos = strlen($_SERVER['SCRIPT_NAME']) - strlen($searched);
172	+ $isScriptIsConsole = ($consolePos === $expectedConsolePos);
173	+ return self::isPhpCliMode() && $isScriptIsConsole;
174	+ }
175	+
176	+ /*
177	+ * String operations
178	+ */
179	+
180	+ /**
181	+ * Multi-byte substr() - works with UTF-8.
182	+ *
183	+ * Calls `mb_substr` if available and falls back to `substr` if it's not.
184	+ *
185	+ * @param string $string
186	+ * @param int $start
187	+ * @param int ... optional length
188	+ * @return string
189	+ * @api
190	+ */
191	+ public static function mb_substr($string, $start)
192	+ {
193	+ $length = func_num_args() > 2
194	+ ? func_get_arg(2)
195	+ : self::mb_strlen($string);
196	+
197	+ if (function_exists('mb_substr')) {
198	+ return mb_substr($string, $start, $length, 'UTF-8');
199	+ }
200	+
201	+ return substr($string, $start, $length);
202	+ }
203	+
204	+ /**
205	+ * Gets the current process ID.
206	+ * Note: If getmypid is disabled, a random ID will be generated once and used throughout the request. There is a
207	+ * small chance that two processes at the same time may generated the same random ID. If you need to rely on the
208	+ * value being 100% unique, then you may need to use `getmypid` directly or some other logic. Eg in CliMulti it is
209	+ * fine to use `getmypid` directly as the logic won't be used if getmypid is disabled...
210	+ * If you are wanting to use the pid to check if the process is running eg using `ps`, then you also have to use
211	+ * getmypid directly.
212	+ *
213	+ * @return int\|null
214	+ */
215	+ public static function getProcessId()
216	+ {
217	+ static $pid;
218	+ if (!isset($pid)) {
219	+ if (Process::isMethodDisabled('getmypid')) {
220	+ $pid = Common::getRandomInt(12);
221	+ } else {
222	+ $pid = getmypid();
223	+ }
224	+ }
225	+
226	+ return $pid;
227	+ }
228	+
229	+ /**
230	+ * Multi-byte strlen() - works with UTF-8
231	+ *
232	+ * Calls `mb_substr` if available and falls back to `substr` if not.
233	+ *
234	+ * @param string $string
235	+ * @return int
236	+ * @api
237	+ */
238	+ public static function mb_strlen($string)
239	+ {
240	+ if (function_exists('mb_strlen')) {
241	+ return mb_strlen($string, 'UTF-8');
242	+ }
243	+
244	+ return strlen($string);
245	+ }
246	+
247	+ /**
248	+ * Multi-byte strtolower() - works with UTF-8.
249	+ *
250	+ * Calls `mb_strtolower` if available and falls back to `strtolower` if not.
251	+ *
252	+ * @param string $string
253	+ * @return string
254	+ * @api
255	+ */
256	+ public static function mb_strtolower($string)
257	+ {
258	+ if (function_exists('mb_strtolower')) {
259	+ return mb_strtolower($string, 'UTF-8');
260	+ }
261	+
262	+ // return unchanged string as using `strtolower` might cause unicode problems
263	+ return $string;
264	+ }
265	+
266	+ /**
267	+ * Multi-byte strtoupper() - works with UTF-8.
268	+ *
269	+ * Calls `mb_strtoupper` if available and falls back to `strtoupper` if not.
270	+ *
271	+ * @param string $string
272	+ * @return string
273	+ * @api
274	+ */
275	+ public static function mb_strtoupper($string)
276	+ {
277	+ if (function_exists('mb_strtoupper')) {
278	+ return mb_strtoupper($string, 'UTF-8');
279	+ }
280	+
281	+ // return unchanged string as using `strtoupper` might cause unicode problems
282	+ return $string;
283	+ }
284	+
285	+ /**
286	+ * Secure wrapper for unserialize, which by default disallows unserializing classes
287	+ *
288	+ * @param string $string String to unserialize
289	+ * @param array $allowedClasses Class names that should be allowed to unserialize
290	+ * @param bool $rethrow Whether to rethrow exceptions or not.
291	+ * @return mixed
292	+ */
293	+ public static function safe_unserialize($string, $allowedClasses = [], $rethrow = false)
294	+ {
295	+ if (PHP_MAJOR_VERSION >= 7) {
296	+ try {
297	+ return unserialize($string, ['allowed_classes' => empty($allowedClasses) ? false : $allowedClasses]);
298	+ } catch (\Throwable $e) {
299	+ if ($rethrow) {
300	+ throw $e;
301	+ }
302	+
303	+ $logger = StaticContainer::get('Psr\Log\LoggerInterface');
304	+ $logger->debug('Unable to unserialize a string: {message} (string = {string})', [
305	+ 'message' => $e->getMessage(),
306	+ 'backtrace' => $e->getTraceAsString(),
307	+ 'string' => $string,
308	+ ]);
309	+ return false;
310	+ }
311	+ }
312	+
313	+ return @unserialize($string);
314	+ }
315	+
316	+ /*
317	+ * Escaping input
318	+ */
319	+
320	+ /**
321	+ * Sanitizes a string to help avoid XSS vulnerabilities.
322	+ *
323	+ * This function is automatically called when {@link getRequestVar()} is called,
324	+ * so you should not normally have to use it.
325	+ *
326	+ * This function should be used when outputting data that isn't escaped and was
327	+ * obtained from the user (for example when using the `\|raw` twig filter on goal names).
328	+ *
329	+ * _NOTE: Sanitized input should not be used directly in an SQL query; SQL placeholders
330	+ * should still be used._
331	+ *
332	+ * Implementation Details
333	+ *
334	+ * - [htmlspecialchars](http://php.net/manual/en/function.htmlspecialchars.php) is used to escape text.
335	+ * - Single quotes are not escaped so Piwik's amazing community will still be
336	+ * Piwik's amazing community.
337	+ * - Use of the `magic_quotes` setting will not break this method.
338	+ * - Boolean, numeric and null values are not modified.
339	+ *
340	+ * @param mixed $value The variable to be sanitized. If an array is supplied, the contents
341	+ * of the array will be sanitized recursively. The keys of the array
342	+ * will also be sanitized.
343	+ * @param bool $alreadyStripslashed Implementation detail, ignore.
344	+ * @throws Exception If `$value` is of an incorrect type.
345	+ * @return mixed The sanitized value.
346	+ * @api
347	+ */
348	+ public static function sanitizeInputValues($value, $alreadyStripslashed = false)
349	+ {
350	+ if (is_numeric($value)) {
351	+ return $value;
352	+ } elseif (is_string($value)) {
353	+ $value = self::sanitizeString($value);
354	+ } elseif (is_array($value)) {
355	+ foreach (array_keys($value) as $key) {
356	+ $newKey = $key;
357	+ $newKey = self::sanitizeInputValues($newKey, $alreadyStripslashed);
358	+ if ($key != $newKey) {
359	+ $value[$newKey] = $value[$key];
360	+ unset($value[$key]);
361	+ }
362	+
363	+ $value[$newKey] = self::sanitizeInputValues($value[$newKey], $alreadyStripslashed);
364	+ }
365	+ } elseif (!is_null($value)
366	+ && !is_bool($value)
367	+ ) {
368	+ throw new Exception("The value to escape has not a supported type. Value = " . var_export($value, true));
369	+ }
370	+ return $value;
371	+ }
372	+
373	+ /**
374	+ * Sanitize a single input value and removes line breaks, tabs and null characters.
375	+ *
376	+ * @param string $value
377	+ * @return string sanitized input
378	+ */
379	+ public static function sanitizeInputValue($value)
380	+ {
381	+ $value = self::sanitizeLineBreaks($value);
382	+ $value = self::sanitizeString($value);
383	+ return $value;
384	+ }
385	+
386	+ /**
387	+ * Sanitize a single input value
388	+ *
389	+ * @param $value
390	+ * @return string
391	+ */
392	+ private static function sanitizeString($value)
393	+ {
394	+ // $_GET and $_REQUEST already urldecode()'d
395	+ // decode
396	+ // note: before php 5.2.7, htmlspecialchars() double encodes &#x hex items
397	+ $value = html_entity_decode($value, self::HTML_ENCODING_QUOTE_STYLE, 'UTF-8');
398	+
399	+ $value = self::sanitizeNullBytes($value);
400	+
401	+ // escape
402	+ $tmp = @htmlspecialchars($value, self::HTML_ENCODING_QUOTE_STYLE, 'UTF-8');
403	+
404	+ // note: php 5.2.5 and above, htmlspecialchars is destructive if input is not UTF-8
405	+ if ($value != '' && $tmp == '') {
406	+ // convert and escape
407	+ $value = utf8_encode($value);
408	+ $tmp = htmlspecialchars($value, self::HTML_ENCODING_QUOTE_STYLE, 'UTF-8');
409	+ return $tmp;
410	+ }
411	+ return $tmp;
412	+ }
413	+
414	+ /**
415	+ * Unsanitizes a single input value and returns the result.
416	+ *
417	+ * @param string $value
418	+ * @return string unsanitized input
419	+ * @api
420	+ */
421	+ public static function unsanitizeInputValue($value)
422	+ {
423	+ return htmlspecialchars_decode($value, self::HTML_ENCODING_QUOTE_STYLE);
424	+ }
425	+
426	+ /**
427	+ * Unsanitizes one or more values and returns the result.
428	+ *
429	+ * This method should be used when you need to unescape data that was obtained from
430	+ * the user.
431	+ *
432	+ * Some data in Piwik is stored sanitized (such as site name). In this case you may
433	+ * have to use this method to unsanitize it in order to, for example, output it in JSON.
434	+ *
435	+ * @param string\|array $value The data to unsanitize. If an array is passed, the
436	+ * array is sanitized recursively. Key values are not unsanitized.
437	+ * @return string\|array The unsanitized data.
438	+ * @api
439	+ */
440	+ public static function unsanitizeInputValues($value)
441	+ {
442	+ if (is_array($value)) {
443	+ $result = array();
444	+ foreach ($value as $key => $arrayValue) {
445	+ $result[$key] = self::unsanitizeInputValues($arrayValue);
446	+ }
447	+ return $result;
448	+ } else {
449	+ return self::unsanitizeInputValue($value);
450	+ }
451	+ }
452	+
453	+ /**
454	+ * @param string $value
455	+ * @return string Line breaks and line carriage removed
456	+ */
457	+ public static function sanitizeLineBreaks($value)
458	+ {
459	+ return str_replace(array("\n", "\r"), '', $value);
460	+ }
461	+
462	+ /**
463	+ * @param string $value
464	+ * @return string Null bytes removed
465	+ */
466	+ public static function sanitizeNullBytes($value)
467	+ {
468	+ return str_replace(array("\0"), '', $value);
469	+ }
470	+
471	+ /**
472	+ * Gets a sanitized request parameter by name from the `$_GET` and `$_POST` superglobals.
473	+ *
474	+ * Use this function to get request parameter values. _NEVER use `$_GET` and `$_POST` directly._
475	+ *
476	+ * If the variable cannot be found, and a default value was not provided, an exception is raised.
477	+ *
478	+ * _See {@link sanitizeInputValues()} to learn more about sanitization._
479	+ *
480	+ * @param string $varName Name of the request parameter to get. By default, we look in `$_GET[$varName]`
481	+ * and `$_POST[$varName]` for the value.
482	+ * @param string\|null $varDefault The value to return if the request parameter cannot be found or has an empty value.
483	+ * @param string\|null $varType Expected type of the request variable. This parameters value must be one of the following:
484	+ * `'array'`, `'int'`, `'integer'`, `'string'`, `'json'`.
485	+ *
486	+ * If `'json'`, the string value will be `json_decode`-d and then sanitized.
487	+ * @param array\|null $requestArrayToUse The array to use instead of `$_GET` and `$_POST`.
488	+ * @throws Exception If the request parameter doesn't exist and there is no default value, or if the request parameter
489	+ * exists but has an incorrect type.
490	+ * @return mixed The sanitized request parameter.
491	+ * @api
492	+ */
493	+ public static function getRequestVar($varName, $varDefault = null, $varType = null, $requestArrayToUse = null)
494	+ {
495	+ if (is_null($requestArrayToUse)) {
496	+ $requestArrayToUse = $_GET + $_POST;
497	+ }
498	+
499	+ $varDefault = self::sanitizeInputValues($varDefault);
500	+ if ($varType === 'int') {
501	+ // settype accepts only integer
502	+ // 'int' is simply a shortcut for 'integer'
503	+ $varType = 'integer';
504	+ }
505	+
506	+ // there is no value $varName in the REQUEST so we try to use the default value
507	+ if (empty($varName)
508	+ \|\| !isset($requestArrayToUse[$varName])
509	+ \|\| (!is_array($requestArrayToUse[$varName])
510	+ && strlen($requestArrayToUse[$varName]) === 0
511	+ )
512	+ ) {
513	+ if (is_null($varDefault)) {
514	+ throw new Exception("The parameter '$varName' isn't set in the Request, and a default value wasn't provided.");
515	+ } else {
516	+ if (!is_null($varType)
517	+ && in_array($varType, array('string', 'integer', 'array'))
518	+ ) {
519	+ settype($varDefault, $varType);
520	+ }
521	+ return $varDefault;
522	+ }
523	+ }
524	+
525	+ // Normal case, there is a value available in REQUEST for the requested varName:
526	+
527	+ // we deal w/ json differently
528	+ if ($varType == 'json') {
529	+ $value = $requestArrayToUse[$varName];
530	+ $value = json_decode($value, $assoc = true);
531	+ return self::sanitizeInputValues($value, $alreadyStripslashed = true);
532	+ }
533	+
534	+ $value = self::sanitizeInputValues($requestArrayToUse[$varName]);
535	+ if (isset($varType)) {
536	+ $ok = false;
537	+
538	+ if ($varType === 'string') {
539	+ if (is_string($value) \|\| is_int($value)) {
540	+ $ok = true;
541	+ } elseif (is_float($value)) {
542	+ $value = Common::forceDotAsSeparatorForDecimalPoint($value);
543	+ $ok = true;
544	+ }
545	+ } elseif ($varType === 'integer') {
546	+ if ($value == (string)(int)$value) {
547	+ $ok = true;
548	+ }
549	+ } elseif ($varType === 'float') {
550	+ $valueToCompare = (string)(float)$value;
551	+ $valueToCompare = Common::forceDotAsSeparatorForDecimalPoint($valueToCompare);
552	+
553	+ if ($value == $valueToCompare) {
554	+ $ok = true;
555	+ }
556	+ } elseif ($varType === 'array') {
557	+ if (is_array($value)) {
558	+ $ok = true;
559	+ }
560	+ } else {
561	+ throw new Exception("\$varType specified is not known. It should be one of the following: array, int, integer, float, string");
562	+ }
563	+
564	+ // The type is not correct
565	+ if ($ok === false) {
566	+ if ($varDefault === null) {
567	+ throw new Exception("The parameter '$varName' doesn't have a correct type, and a default value wasn't provided.");
568	+ } // we return the default value with the good type set
569	+ else {
570	+ settype($varDefault, $varType);
571	+ return $varDefault;
572	+ }
573	+ }
574	+ settype($value, $varType);
575	+ }
576	+
577	+ return $value;
578	+ }
579	+
580	+ /*
581	+ * Generating unique strings
582	+ */
583	+
584	+ /**
585	+ * Generates a random integer
586	+ *
587	+ * @param int $min
588	+ * @param null\|int $max Defaults to max int value
589	+ * @return int\|null
590	+ */
591	+ public static function getRandomInt($min = 0, $max = null)
592	+ {
593	+ $rand = null;
594	+
595	+ if (function_exists('random_int')) {
596	+ try {
597	+ if (!isset($max)) {
598	+ $max = PHP_INT_MAX;
599	+ }
600	+ $rand = random_int($min, $max);
601	+ } catch (Exception $e) {
602	+ // If none of the crypto sources are available, an Exception will be thrown.
603	+ $rand = null;
604	+ }
605	+ }
606	+
607	+ if (!isset($rand)) {
608	+ if (function_exists('mt_rand')) {
609	+ if (!isset($max)) {
610	+ $max = mt_getrandmax();
611	+ }
612	+ $rand = mt_rand($min, $max);
613	+ } else {
614	+ if (!isset($max)) {
615	+ $max = getrandmax();
616	+ }
617	+
618	+ $rand = rand($min, $max);
619	+ }
620	+ }
621	+
622	+ return $rand;
623	+ }
624	+
625	+ /**
626	+ * Returns a 32 characters long uniq ID
627	+ *
628	+ * @return string 32 chars
629	+ */
630	+ public static function generateUniqId()
631	+ {
632	+ $rand = self::getRandomInt();
633	+
634	+ return md5(uniqid($rand, true));
635	+ }
636	+
637	+ /**
638	+ * Configurable hash() algorithm (defaults to md5)
639	+ *
640	+ * @param string $str String to be hashed
641	+ * @param bool $raw_output
642	+ * @return string Hash string
643	+ */
644	+ public static function hash($str, $raw_output = false)
645	+ {
646	+ static $hashAlgorithm = null;
647	+
648	+ if (is_null($hashAlgorithm)) {
649	+ $hashAlgorithm = @Config::getInstance()->General['hash_algorithm'];
650	+ }
651	+
652	+ if ($hashAlgorithm) {
653	+ $hash = @hash($hashAlgorithm, $str, $raw_output);
654	+ if ($hash !== false) {
655	+ return $hash;
656	+ }
657	+ }
658	+
659	+ return md5($str, $raw_output);
660	+ }
661	+
662	+ /**
663	+ * Generate random string.
664	+ *
665	+ * @param int $length string length
666	+ * @param string $alphabet characters allowed in random string
667	+ * @return string random string with given length
668	+ */
669	+ public static function getRandomString($length = 16, $alphabet = "abcdefghijklmnoprstuvwxyz0123456789")
670	+ {
671	+ $chars = $alphabet;
672	+ $str = '';
673	+
674	+ for ($i = 0; $i < $length; $i++) {
675	+ $rand_key = self::getRandomInt(0, strlen($chars) - 1);
676	+ $str .= substr($chars, $rand_key, 1);
677	+ }
678	+
679	+ return str_shuffle($str);
680	+ }
681	+
682	+ /*
683	+ * Conversions
684	+ */
685	+
686	+ /**
687	+ * Convert hexadecimal representation into binary data.
688	+ * !! Will emit warning if input string is not hex!!
689	+ *
690	+ * @see http://php.net/bin2hex
691	+ *
692	+ * @param string $str Hexadecimal representation
693	+ * @return string
694	+ */
695	+ public static function hex2bin($str)
696	+ {
697	+ return pack("H*", $str);
698	+ }
699	+
700	+ /**
701	+ * This function will convert the input string to the binary representation of the ID
702	+ * but it will throw an Exception if the specified input ID is not correct
703	+ *
704	+ * This is used when building segments containing visitorId which could be an invalid string
705	+ * therefore throwing Unexpected PHP error [pack(): Type H: illegal hex digit i] severity [E_WARNING]
706	+ *
707	+ * It would be simply to silent fail the pack() call above but in all other cases, we don't expect an error,
708	+ * so better be safe and get the php error when something unexpected is happening
709	+ * @param string $id
710	+ * @throws Exception
711	+ * @return string binary string
712	+ */
713	+ public static function convertVisitorIdToBin($id)
714	+ {
715	+ if (strlen($id) !== Tracker::LENGTH_HEX_ID_STRING
716	+ \|\| @bin2hex(self::hex2bin($id)) != $id
717	+ ) {
718	+ throw new Exception("visitorId is expected to be a " . Tracker::LENGTH_HEX_ID_STRING . " hex char string");
719	+ }
720	+
721	+ return self::hex2bin($id);
722	+ }
723	+
724	+ /**
725	+ * Converts a User ID string to the Visitor ID Binary representation.
726	+ *
727	+ * @param $userId
728	+ * @return string
729	+ */
730	+ public static function convertUserIdToVisitorIdBin($userId)
731	+ {
732	+ require_once PIWIK_INCLUDE_PATH . '/libs/PiwikTracker/PiwikTracker.php';
733	+ $userIdHashed = \PiwikTracker::getUserIdHashed($userId);
734	+
735	+ return self::convertVisitorIdToBin($userIdHashed);
736	+ }
737	+
738	+ /**
739	+ * JSON encode wrapper
740	+ * - missing or broken in some php 5.x versions
741	+ *
742	+ * @param mixed $value
743	+ * @return string
744	+ * @deprecated
745	+ */
746	+ public static function json_encode($value)
747	+ {
748	+ return @json_encode($value);
749	+ }
750	+
751	+ /**
752	+ * JSON decode wrapper
753	+ * - missing or broken in some php 5.x versions
754	+ *
755	+ * @param string $json
756	+ * @param bool $assoc
757	+ * @return mixed
758	+ * @deprecated
759	+ */
760	+ public static function json_decode($json, $assoc = false)
761	+ {
762	+ return json_decode($json, $assoc);
763	+ }
764	+
765	+ /**
766	+ * Detects whether an error occurred during the last json encode/decode.
767	+ * @return bool
768	+ */
769	+ public static function hasJsonErrorOccurred()
770	+ {
771	+ return json_last_error() != JSON_ERROR_NONE;
772	+ }
773	+
774	+ /**
775	+ * Returns a human readable error message in case an error occcurred during the last json encode/decode.
776	+ * Returns an empty string in case there was no error.
777	+ *
778	+ * @return string
779	+ */
780	+ public static function getLastJsonError()
781	+ {
782	+ switch (json_last_error()) {
783	+ case JSON_ERROR_NONE:
784	+ return '';
785	+ case JSON_ERROR_DEPTH:
786	+ return 'Maximum stack depth exceeded';
787	+ case JSON_ERROR_STATE_MISMATCH:
788	+ return 'Underflow or the modes mismatch';
789	+ case JSON_ERROR_CTRL_CHAR:
790	+ return 'Unexpected control character found';
791	+ case JSON_ERROR_SYNTAX:
792	+ return 'Syntax error, malformed JSON';
793	+ case JSON_ERROR_UTF8:
794	+ return 'Malformed UTF-8 characters, possibly incorrectly encoded';
795	+ }
796	+
797	+ return 'Unknown error';
798	+ }
799	+
800	+ public static function stringEndsWith($haystack, $needle)
801	+ {
802	+ if ('' === $needle) {
803	+ return true;
804	+ }
805	+
806	+ $lastCharacters = substr($haystack, -strlen($needle));
807	+
808	+ return $lastCharacters === $needle;
809	+ }
810	+
811	+ /**
812	+ * Returns the list of parent classes for the given class.
813	+ *
814	+ * @param string $class A class name.
815	+ * @return string[] The list of parent classes in order from highest ancestor to the descended class.
816	+ */
817	+ public static function getClassLineage($class)
818	+ {
819	+ $classes = array_merge(array($class), array_values(class_parents($class, $autoload = false)));
820	+
821	+ return array_reverse($classes);
822	+ }
823	+
824	+ /*
825	+ * DataFiles
826	+ */
827	+
828	+ /**
829	+ * Returns list of continent codes
830	+ *
831	+ * @see core/DataFiles/Countries.php
832	+ *
833	+ * @return array Array of 3 letter continent codes
834	+ *
835	+ * @deprecated Use Piwik\Intl\Data\Provider\RegionDataProvider instead.
836	+ * @see \Piwik\Intl\Data\Provider\RegionDataProvider::getContinentList()
837	+ */
838	+ public static function getContinentsList()
839	+ {
840	+ /** @var RegionDataProvider $dataProvider */
841	+ $dataProvider = StaticContainer::get('Piwik\Intl\Data\Provider\RegionDataProvider');
842	+ return $dataProvider->getContinentList();
843	+ }
844	+
845	+ /**
846	+ * Returns list of valid country codes
847	+ *
848	+ * @see core/DataFiles/Countries.php
849	+ *
850	+ * @param bool $includeInternalCodes
851	+ * @return array Array of (2 letter ISO codes => 3 letter continent code)
852	+ *
853	+ * @deprecated Use Piwik\Intl\Data\Provider\RegionDataProvider instead.
854	+ * @see \Piwik\Intl\Data\Provider\RegionDataProvider::getCountryList()
855	+ */
856	+ public static function getCountriesList($includeInternalCodes = false)
857	+ {
858	+ /** @var RegionDataProvider $dataProvider */
859	+ $dataProvider = StaticContainer::get('Piwik\Intl\Data\Provider\RegionDataProvider');
860	+ return $dataProvider->getCountryList($includeInternalCodes);
861	+ }
862	+
863	+ /**
864	+ * Returns the list of valid language codes.
865	+ *
866	+ * See [core/DataFiles/Languages.php](https://github.com/piwik/piwik/blob/master/core/DataFiles/Languages.php).
867	+ *
868	+ * @return array Array of two letter ISO codes mapped with their associated language names (in English). E.g.
869	+ * `array('en' => 'English', 'ja' => 'Japanese')`.
870	+ * @api
871	+ *
872	+ * @deprecated Use Piwik\Intl\Data\Provider\LanguageDataProvider instead.
873	+ * @see \Piwik\Intl\Data\Provider\LanguageDataProvider::getLanguageList()
874	+ */
875	+ public static function getLanguagesList()
876	+ {
877	+ /** @var LanguageDataProvider $dataProvider */
878	+ $dataProvider = StaticContainer::get('Piwik\Intl\Data\Provider\LanguageDataProvider');
879	+ return $dataProvider->getLanguageList();
880	+ }
881	+
882	+ /**
883	+ * Returns a list of language to country mappings.
884	+ *
885	+ * See [core/DataFiles/LanguageToCountry.php](https://github.com/piwik/piwik/blob/master/core/DataFiles/LanguageToCountry.php).
886	+ *
887	+ * @return array Array of two letter ISO language codes mapped with two letter ISO country codes:
888	+ * `array('fr' => 'fr') // French => France`
889	+ * @api
890	+ *
891	+ * @deprecated Use Piwik\Intl\Data\Provider\LanguageDataProvider instead.
892	+ * @see \Piwik\Intl\Data\Provider\LanguageDataProvider::getLanguageToCountryList()
893	+ */
894	+ public static function getLanguageToCountryList()
895	+ {
896	+ /** @var LanguageDataProvider $dataProvider */
897	+ $dataProvider = StaticContainer::get('Piwik\Intl\Data\Provider\LanguageDataProvider');
898	+ return $dataProvider->getLanguageToCountryList();
899	+ }
900	+
901	+ /**
902	+ * Returns list of provider names
903	+ *
904	+ * @see core/DataFiles/Providers.php
905	+ *
906	+ * @return array Array of ( dnsName => providerName )
907	+ */
908	+ public static function getProviderNames()
909	+ {
910	+ require_once PIWIK_INCLUDE_PATH . '/core/DataFiles/Providers.php';
911	+
912	+ $providers = $GLOBALS['Piwik_ProviderNames'];
913	+ return $providers;
914	+ }
915	+
916	+ /*
917	+ * Language, country, continent
918	+ */
919	+
920	+ /**
921	+ * Returns the browser language code, eg. "en-gb,en;q=0.5"
922	+ *
923	+ * @param string\|null $browserLang Optional browser language, otherwise taken from the request header
924	+ * @return string
925	+ */
926	+ public static function getBrowserLanguage($browserLang = null)
927	+ {
928	+ static $replacementPatterns = array(
929	+ // extraneous bits of RFC 3282 that we ignore
930	+ '/(\\\\.)/', // quoted-pairs
931	+ '/(\s+)/', // CFWcS white space
932	+ '/($[^)]*$)/', // CFWS comments
933	+ '/(;q=[0-9.]+)/', // quality
934	+
935	+ // found in the LANG environment variable
936	+ '/\.(.*)/', // charset (e.g., en_CA.UTF-8)
937	+ '/^C$/', // POSIX 'C' locale
938	+ );
939	+
940	+ if (is_null($browserLang)) {
941	+ $browserLang = self::sanitizeInputValues(@$_SERVER['HTTP_ACCEPT_LANGUAGE']);
942	+ if (empty($browserLang) && self::isPhpCliMode()) {
943	+ $browserLang = @getenv('LANG');
944	+ }
945	+ }
946	+
947	+ if (empty($browserLang)) {
948	+ // a fallback might be to infer the language in HTTP_USER_AGENT (i.e., localized build)
949	+ $browserLang = "";
950	+ } else {
951	+ // language tags are case-insensitive per HTTP/1.1 s3.10 but the region may be capitalized per ISO3166-1;
952	+ // underscores are not permitted per RFC 4646 or 4647 (which obsolete RFC 1766 and 3066),
953	+ // but we guard against a bad user agent which naively uses its locale
954	+ $browserLang = strtolower(str_replace('_', '-', $browserLang));
955	+
956	+ // filters
957	+ $browserLang = preg_replace($replacementPatterns, '', $browserLang);
958	+
959	+ $browserLang = preg_replace('/((^\|,)chrome:.*)/', '', $browserLang, 1); // Firefox bug
960	+ $browserLang = preg_replace('/(,)(?:en-securid,)\|(?:(^\|,)en-securid(,\|$))/', '$1', $browserLang, 1); // unregistered language tag
961	+
962	+ $browserLang = str_replace('sr-sp', 'sr-rs', $browserLang); // unofficial (proposed) code in the wild
963	+ }
964	+
965	+ return $browserLang;
966	+ }
967	+
968	+ /**
969	+ * Returns the visitor country based on the Browser 'accepted language'
970	+ * information, but provides a hook for geolocation via IP address.
971	+ *
972	+ * @param string $lang browser lang
973	+ * @param bool $enableLanguageToCountryGuess If set to true, some assumption will be made and detection guessed more often, but accuracy could be affected
974	+ * @param string $ip
975	+ * @return string 2 letter ISO code
976	+ */
977	+ public static function getCountry($lang, $enableLanguageToCountryGuess, $ip)
978	+ {
979	+ if (empty($lang) \|\| strlen($lang) < 2 \|\| $lang == self::LANGUAGE_CODE_INVALID) {
980	+ return self::LANGUAGE_CODE_INVALID;
981	+ }
982	+
983	+ /** @var RegionDataProvider $dataProvider */
984	+ $dataProvider = StaticContainer::get('Piwik\Intl\Data\Provider\RegionDataProvider');
985	+
986	+ $validCountries = $dataProvider->getCountryList();
987	+
988	+ return self::extractCountryCodeFromBrowserLanguage($lang, $validCountries, $enableLanguageToCountryGuess);
989	+ }
990	+
991	+ /**
992	+ * Returns list of valid country codes
993	+ *
994	+ * @param string $browserLanguage
995	+ * @param array $validCountries Array of valid countries
996	+ * @param bool $enableLanguageToCountryGuess (if true, will guess country based on language that lacks region information)
997	+ * @return array Array of 2 letter ISO codes
998	+ */
999	+ public static function extractCountryCodeFromBrowserLanguage($browserLanguage, $validCountries, $enableLanguageToCountryGuess)
1000	+ {
1001	+ /** @var LanguageDataProvider $dataProvider */
1002	+ $dataProvider = StaticContainer::get('Piwik\Intl\Data\Provider\LanguageDataProvider');
1003	+
1004	+ $langToCountry = $dataProvider->getLanguageToCountryList();
1005	+
1006	+ if ($enableLanguageToCountryGuess) {
1007	+ if (preg_match('/^([a-z]{2,3})(?:,\|;\|$)/', $browserLanguage, $matches)) {
1008	+ // match language (without region) to infer the country of origin
1009	+ if (array_key_exists($matches[1], $langToCountry)) {
1010	+ return $langToCountry[$matches[1]];
1011	+ }
1012	+ }
1013	+ }
1014	+
1015	+ if (!empty($validCountries) && preg_match_all('/[-]([a-z]{2})/', $browserLanguage, $matches, PREG_SET_ORDER)) {
1016	+ foreach ($matches as $parts) {
1017	+ // match location; we don't make any inferences from the language
1018	+ if (array_key_exists($parts[1], $validCountries)) {
1019	+ return $parts[1];
1020	+ }
1021	+ }
1022	+ }
1023	+ return self::LANGUAGE_CODE_INVALID;
1024	+ }
1025	+
1026	+ /**
1027	+ * Returns the language and region string, based only on the Browser 'accepted language' information.
1028	+ * * The language tag is defined by ISO 639-1
1029	+ *
1030	+ * @param string $browserLanguage Browser's accepted langauge header
1031	+ * @param array $validLanguages array of valid language codes
1032	+ * @return string 2 letter ISO 639 code 'es' (Spanish)
1033	+ */
1034	+ public static function extractLanguageCodeFromBrowserLanguage($browserLanguage, $validLanguages = array())
1035	+ {
1036	+ $validLanguages = self::checkValidLanguagesIsSet($validLanguages);
1037	+ $languageRegionCode = self::extractLanguageAndRegionCodeFromBrowserLanguage($browserLanguage, $validLanguages);
1038	+
1039	+ if (strlen($languageRegionCode) == 2) {
1040	+ $languageCode = $languageRegionCode;
1041	+ } else {
1042	+ $languageCode = substr($languageRegionCode, 0, 2);
1043	+ }
1044	+ if (in_array($languageCode, $validLanguages)) {
1045	+ return $languageCode;
1046	+ }
1047	+ return self::LANGUAGE_CODE_INVALID;
1048	+ }
1049	+
1050	+ /**
1051	+ * Returns the language and region string, based only on the Browser 'accepted language' information.
1052	+ * * The language tag is defined by ISO 639-1
1053	+ * * The region tag is defined by ISO 3166-1
1054	+ *
1055	+ * @param string $browserLanguage Browser's accepted langauge header
1056	+ * @param array $validLanguages array of valid language codes. Note that if the array includes "fr" then it will consider all regional variants of this language valid, such as "fr-ca" etc.
1057	+ * @return string 2 letter ISO 639 code 'es' (Spanish) or if found, includes the region as well: 'es-ar'
1058	+ */
1059	+ public static function extractLanguageAndRegionCodeFromBrowserLanguage($browserLanguage, $validLanguages = array())
1060	+ {
1061	+ $validLanguages = self::checkValidLanguagesIsSet($validLanguages);
1062	+
1063	+ if (!preg_match_all('/(?:^\|,)([a-z]{2,3})([-][a-z]{2})?/', $browserLanguage, $matches, PREG_SET_ORDER)) {
1064	+ return self::LANGUAGE_CODE_INVALID;
1065	+ }
1066	+ foreach ($matches as $parts) {
1067	+ $langIso639 = $parts[1];
1068	+ if (empty($langIso639)) {
1069	+ continue;
1070	+ }
1071	+
1072	+ // If a region tag is found eg. "fr-ca"
1073	+ if (count($parts) == 3) {
1074	+ $regionIso3166 = $parts[2]; // eg. "-ca"
1075	+
1076	+ if (in_array($langIso639 . $regionIso3166, $validLanguages)) {
1077	+ return $langIso639 . $regionIso3166;
1078	+ }
1079	+
1080	+ if (in_array($langIso639, $validLanguages)) {
1081	+ return $langIso639 . $regionIso3166;
1082	+ }
1083	+ }
1084	+ // eg. "fr" or "es"
1085	+ if (in_array($langIso639, $validLanguages)) {
1086	+ return $langIso639;
1087	+ }
1088	+ }
1089	+ return self::LANGUAGE_CODE_INVALID;
1090	+ }
1091	+
1092	+ /**
1093	+ * Returns the continent of a given country
1094	+ *
1095	+ * @param string $country 2 letters iso code
1096	+ *
1097	+ * @return string Continent (3 letters code : afr, asi, eur, amn, ams, oce)
1098	+ */
1099	+ public static function getContinent($country)
1100	+ {
1101	+ /** @var RegionDataProvider $dataProvider */
1102	+ $dataProvider = StaticContainer::get('Piwik\Intl\Data\Provider\RegionDataProvider');
1103	+
1104	+ $countryList = $dataProvider->getCountryList();
1105	+
1106	+ if ($country == 'ti') {
1107	+ $country = 'cn';
1108	+ }
1109	+
1110	+ return isset($countryList[$country]) ? $countryList[$country] : 'unk';
1111	+ }
1112	+
1113	+ /*
1114	+ * Campaign
1115	+ */
1116	+
1117	+ /**
1118	+ * Returns the list of Campaign parameter names that will be read to classify
1119	+ * a visit as coming from a Campaign
1120	+ *
1121	+ * @return array array(
1122	+ * 0 => array( ... ) // campaign names parameters
1123	+ * 1 => array( ... ) // campaign keyword parameters
1124	+ * );
1125	+ */
1126	+ public static function getCampaignParameters()
1127	+ {
1128	+ $return = array(
1129	+ Config::getInstance()->Tracker['campaign_var_name'],
1130	+ Config::getInstance()->Tracker['campaign_keyword_var_name'],
1131	+ );
1132	+
1133	+ foreach ($return as &$list) {
1134	+ if (strpos($list, ',') !== false) {
1135	+ $list = explode(',', $list);
1136	+ } else {
1137	+ $list = array($list);
1138	+ }
1139	+ $list = array_map('trim', $list);
1140	+ }
1141	+
1142	+ return $return;
1143	+ }
1144	+
1145	+ /*
1146	+ * Referrer
1147	+ */
1148	+
1149	+ /**
1150	+ * Returns a string with a comma separated list of placeholders for use in an SQL query. Used mainly
1151	+ * to fill the `IN (...)` part of a query.
1152	+ *
1153	+ * @param array\|string $fields The names of the mysql table fields to bind, e.g.
1154	+ * `array(fieldName1, fieldName2, fieldName3)`.
1155	+ *
1156	+ * _Note: The content of the array isn't important, just its length._
1157	+ * @return string The placeholder string, e.g. `"?, ?, ?"`.
1158	+ * @api
1159	+ */
1160	+ public static function getSqlStringFieldsArray($fields)
1161	+ {
1162	+ if (is_string($fields)) {
1163	+ $fields = array($fields);
1164	+ }
1165	+ $count = count($fields);
1166	+ if ($count == 0) {
1167	+ return "''";
1168	+ }
1169	+ return '?' . str_repeat(',?', $count - 1);
1170	+ }
1171	+
1172	+ /**
1173	+ * Force the separator for decimal point to be a dot. See https://github.com/piwik/piwik/issues/6435
1174	+ * If for instance a German locale is used it would be a comma otherwise.
1175	+ *
1176	+ * @param float\|string $value
1177	+ * @return string
1178	+ */
1179	+ public static function forceDotAsSeparatorForDecimalPoint($value)
1180	+ {
1181	+ if (null === $value \|\| false === $value) {
1182	+ return $value;
1183	+ }
1184	+
1185	+ return str_replace(',', '.', $value);
1186	+ }
1187	+
1188	+ /**
1189	+ * Sets outgoing header.
1190	+ *
1191	+ * @param string $header The header.
1192	+ * @param bool $replace Whether to replace existing or not.
1193	+ */
1194	+ public static function sendHeader($header, $replace = true)
1195	+ {
1196	+ // don't send header in CLI mode
1197	+ if (!Common::isPhpCliMode() and !headers_sent()) {
1198	+ header($header, $replace);
1199	+ }
1200	+ }
1201	+
1202	+ /**
1203	+ * Strips outgoing header.
1204	+ *
1205	+ * @param string $name The header name.
1206	+ */
1207	+ public static function stripHeader($name)
1208	+ {
1209	+ // don't strip header in CLI mode
1210	+ if (!Common::isPhpCliMode() and !headers_sent()) {
1211	+ header_remove($name);
1212	+ }
1213	+ }
1214	+
1215	+ /**
1216	+ * Sends the given response code if supported.
1217	+ *
1218	+ * @param int $code Eg 204
1219	+ *
1220	+ * @throws Exception
1221	+ */
1222	+ public static function sendResponseCode($code)
1223	+ {
1224	+ $messages = array(
1225	+ 200 => 'Ok',
1226	+ 204 => 'No Response',
1227	+ 301 => 'Moved Permanently',
1228	+ 302 => 'Found',
1229	+ 304 => 'Not Modified',
1230	+ 400 => 'Bad Request',
1231	+ 401 => 'Unauthorized',
1232	+ 403 => 'Forbidden',
1233	+ 404 => 'Not Found',
1234	+ 500 => 'Internal Server Error',
1235	+ 503 => 'Service Unavailable',
1236	+ );
1237	+
1238	+ if (!array_key_exists($code, $messages)) {
1239	+ throw new Exception('Response code not supported: ' . $code);
1240	+ }
1241	+
1242	+ if (strpos(PHP_SAPI, '-fcgi') === false) {
1243	+ $key = 'HTTP/1.1';
1244	+
1245	+ if (array_key_exists('SERVER_PROTOCOL', $_SERVER)
1246	+ && strlen($_SERVER['SERVER_PROTOCOL']) < 15
1247	+ && strlen($_SERVER['SERVER_PROTOCOL']) > 1) {
1248	+ $key = $_SERVER['SERVER_PROTOCOL'];
1249	+ }
1250	+ } else {
1251	+ // FastCGI
1252	+ $key = 'Status:';
1253	+ }
1254	+
1255	+ $message = $messages[$code];
1256	+ Common::sendHeader($key . ' ' . $code . ' ' . $message);
1257	+ }
1258	+
1259	+ /**
1260	+ * Returns the ID of the current LocationProvider (see UserCountry plugin code) from
1261	+ * the Tracker cache.
1262	+ */
1263	+ public static function getCurrentLocationProviderId()
1264	+ {
1265	+ $cache = TrackerCache::getCacheGeneral();
1266	+ return empty($cache['currentLocationProviderId'])
1267	+ ? DefaultProvider::ID
1268	+ : $cache['currentLocationProviderId'];
1269	+ }
1270	+
1271	+ /**
1272	+ * Marks an orphaned object for garbage collection.
1273	+ *
1274	+ * For more information: {@link https://github.com/piwik/piwik/issues/374}
1275	+ * @param mixed $var The object to destroy.
1276	+ * @api
1277	+ */
1278	+ public static function destroy(&$var)
1279	+ {
1280	+ if (is_object($var) && method_exists($var, '__destruct')) {
1281	+ $var->__destruct();
1282	+ }
1283	+ unset($var);
1284	+ $var = null;
1285	+ }
1286	+
1287	+ /**
1288	+ * @deprecated Use the logger directly instead.
1289	+ */
1290	+ public static function printDebug($info = '')
1291	+ {
1292	+ if (is_object($info)) {
1293	+ $info = var_export($info, true);
1294	+ }
1295	+
1296	+ $logger = StaticContainer::get('Psr\Log\LoggerInterface');
1297	+ if (is_array($info) \|\| is_object($info)) {
1298	+ $out = var_export($info, true);
1299	+ $logger->debug($out);
1300	+ } else {
1301	+ $logger->debug($info);
1302	+ }
1303	+ }
1304	+
1305	+ /**
1306	+ * Returns true if the request is an AJAX request.
1307	+ *
1308	+ * @return bool
1309	+ */
1310	+ public static function isXmlHttpRequest()
1311	+ {
1312	+ return isset($_SERVER['HTTP_X_REQUESTED_WITH'])
1313	+ && (strtolower($_SERVER['HTTP_X_REQUESTED_WITH']) === 'xmlhttprequest');
1314	+ }
1315	+
1316	+ /**
1317	+ * @param $validLanguages
1318	+ * @return array
1319	+ */
1320	+ protected static function checkValidLanguagesIsSet($validLanguages)
1321	+ {
1322	+ /** @var LanguageDataProvider $dataProvider */
1323	+ $dataProvider = StaticContainer::get('Piwik\Intl\Data\Provider\LanguageDataProvider');
1324	+
1325	+ if (empty($validLanguages)) {
1326	+ $validLanguages = array_keys($dataProvider->getLanguageList());
1327	+ return $validLanguages;
1328	+ }
1329	+ return $validLanguages;
1330	+ }
1331	+ }

app/core/Composer/ScriptHandler.php ADDED Viewed

	@@ -0,0 +1,47 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ */
8	+
9	+ namespace Piwik\Composer;
10	+
11	+ /**
12	+ * Scripts executed before/after Composer install and update.
13	+ *
14	+ * We use this PHP class because setting the bash scripts directly in composer.json breaks
15	+ * Composer on Windows systems.
16	+ */
17	+ class ScriptHandler
18	+ {
19	+ private static function isPhp7orLater()
20	+ {
21	+ return version_compare('7.0.0-dev', PHP_VERSION) < 1;
22	+ }
23	+
24	+ public static function cleanXhprof()
25	+ {
26	+ if (! is_dir('vendor/facebook/xhprof/extension')) {
27	+ return;
28	+ }
29	+
30	+ if (!self::isPhp7orLater()) {
31	+ // doesn't work with PHP 7 at the moment
32	+ passthru('misc/composer/clean-xhprof.sh');
33	+ }
34	+ }
35	+
36	+ public static function buildXhprof()
37	+ {
38	+ if (! is_dir('vendor/facebook/xhprof/extension')) {
39	+ return;
40	+ }
41	+
42	+
43	+ if (!self::isPhp7orLater()) {
44	+ passthru('misc/composer/clean-xhprof.sh');
45	+ }
46	+ }
47	+ }

app/core/Concurrency/DistributedList.php ADDED Viewed

	@@ -0,0 +1,171 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ */
8	+ namespace Piwik\Concurrency;
9	+
10	+ use Piwik\Common;
11	+ use Piwik\Container\StaticContainer;
12	+ use Piwik\Option;
13	+ use Psr\Log\LoggerInterface;
14	+
15	+ /**
16	+ * Manages a simple distributed list stored in an Option. No locking occurs, so the list
17	+ * is not thread safe, and should only be used for use cases where atomicity is not
18	+ * important.
19	+ *
20	+ * The list of items is serialized and stored in an Option. Items are converted to string
21	+ * before being persisted, so it is not expected to unserialize objects.
22	+ */
23	+ class DistributedList
24	+ {
25	+ /**
26	+ * The name of the option to store the list in.
27	+ *
28	+ * @var string
29	+ */
30	+ private $optionName;
31	+
32	+ /**
33	+ * @var LoggerInterface
34	+ */
35	+ private $logger;
36	+
37	+ /**
38	+ * Constructor.
39	+ *
40	+ * @param string $optionName
41	+ */
42	+ public function __construct($optionName, LoggerInterface $logger = null)
43	+ {
44	+ $this->optionName = $optionName;
45	+ $this->logger = $logger ?: StaticContainer::get('Psr\Log\LoggerInterface');
46	+ }
47	+
48	+ /**
49	+ * Queries the option table and returns all items in this list.
50	+ *
51	+ * @return array
52	+ */
53	+ public function getAll()
54	+ {
55	+ $result = $this->getListOptionValue();
56	+
57	+ foreach ($result as $key => $item) {
58	+ // remove non-array items (unexpected state, though can happen when upgrading from an old Piwik)
59	+ if (is_array($item)) {
60	+ $this->logger->info("Found array item in DistributedList option value '{name}': {data}", array(
61	+ 'name' => $this->optionName,
62	+ 'data' => var_export($result, true)
63	+ ));
64	+
65	+ unset($result[$key]);
66	+ }
67	+ }
68	+
69	+ return $result;
70	+ }
71	+
72	+ /**
73	+ * Sets the contents of the list in the option table.
74	+ *
75	+ * @param string[] $items
76	+ */
77	+ public function setAll($items)
78	+ {
79	+ foreach ($items as $key => &$item) {
80	+ if (is_array($item)) {
81	+ throw new \InvalidArgumentException("Array item encountered in DistributedList::setAll() [ key = $key ].");
82	+ } else {
83	+ $item = (string)$item;
84	+ }
85	+ }
86	+
87	+ Option::set($this->optionName, serialize($items));
88	+ }
89	+
90	+ /**
91	+ * Adds one or more items to the list in the option table.
92	+ *
93	+ * @param string\|array $item
94	+ */
95	+ public function add($item)
96	+ {
97	+ $allItems = $this->getAll();
98	+ if (is_array($item)) {
99	+ $allItems = array_merge($allItems, $item);
100	+ } else {
101	+ $allItems[] = $item;
102	+ }
103	+
104	+ $this->setAll($allItems);
105	+ }
106	+
107	+ /**
108	+ * Removes one or more items by value from the list in the option table.
109	+ *
110	+ * Does not preserve array keys.
111	+ *
112	+ * @param string\|array $items
113	+ */
114	+ public function remove($items)
115	+ {
116	+ if (!is_array($items)) {
117	+ $items = array($items);
118	+ }
119	+
120	+ $allItems = $this->getAll();
121	+
122	+ foreach ($items as $item) {
123	+ $existingIndex = array_search($item, $allItems);
124	+ if ($existingIndex === false) {
125	+ return;
126	+ }
127	+
128	+ unset($allItems[$existingIndex]);
129	+ }
130	+
131	+ $this->setAll(array_values($allItems));
132	+ }
133	+
134	+ /**
135	+ * Removes one or more items by index from the list in the option table.
136	+ *
137	+ * Does not preserve array keys.
138	+ *
139	+ * @param int[]\|int $indices
140	+ */
141	+ public function removeByIndex($indices)
142	+ {
143	+ if (!is_array($indices)) {
144	+ $indices = array($indices);
145	+ }
146	+
147	+ $indices = array_unique($indices);
148	+
149	+ $allItems = $this->getAll();
150	+ foreach ($indices as $index) {
151	+ unset($allItems[$index]);
152	+ }
153	+
154	+ $this->setAll(array_values($allItems));
155	+ }
156	+
157	+ protected function getListOptionValue()
158	+ {
159	+ Option::clearCachedOption($this->optionName);
160	+ $array = Option::get($this->optionName);
161	+
162	+ $result = array();
163	+ if ($array
164	+ && ($array = Common::safe_unserialize($array))
165	+ && count($array)
166	+ ) {
167	+ $result = $array;
168	+ }
169	+ return $result;
170	+ }
171	+ }

app/core/Concurrency/Lock.php ADDED Viewed

	@@ -0,0 +1,138 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link http://piwik.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik\Concurrency;
10	+
11	+ use Piwik\Common;
12	+
13	+ class Lock
14	+ {
15	+ const MAX_KEY_LEN = 70;
16	+
17	+ /**
18	+ * @var LockBackend
19	+ */
20	+ private $backend;
21	+
22	+ private $lockKeyStart;
23	+
24	+ private $lockKey = null;
25	+ private $lockValue = null;
26	+ private $defaultTtl = null;
27	+
28	+ public function __construct(LockBackend $backend, $lockKeyStart, $defaultTtl = null)
29	+ {
30	+ $this->backend = $backend;
31	+ $this->lockKeyStart = $lockKeyStart;
32	+ $this->lockKey = $this->lockKeyStart;
33	+ $this->defaultTtl = $defaultTtl;
34	+ }
35	+
36	+ public function reexpireLock()
37	+ {
38	+ $this->expireLock($this->defaultTtl);
39	+ }
40	+
41	+ public function getNumberOfAcquiredLocks()
42	+ {
43	+ return count($this->getAllAcquiredLockKeys());
44	+ }
45	+
46	+ public function getAllAcquiredLockKeys()
47	+ {
48	+ return $this->backend->getKeysMatchingPattern($this->lockKeyStart . '*');
49	+ }
50	+
51	+ public function execute($id, $callback)
52	+ {
53	+ $i = 0;
54	+ while (!$this->acquireLock($id)) {
55	+ $i++;
56	+ usleep( 100 * 1000 ); // 100ms
57	+ if ($i > 50) { // give up after 5seconds (50 * 100ms)
58	+ throw new \Exception('Could not get the lock for ID: ' . $id);
59	+ }
60	+ };
61	+ try {
62	+ return $callback();
63	+ } finally {
64	+ $this->unlock();
65	+ }
66	+ }
67	+
68	+ public function acquireLock($id, $ttlInSeconds = 60)
69	+ {
70	+ $this->lockKey = $this->lockKeyStart . $id;
71	+
72	+ if (Common::mb_strlen($this->lockKey) > self::MAX_KEY_LEN) {
73	+ // Lock key might be too long for DB column, so we hash it but leave the start of the original as well
74	+ // to make it more readable
75	+ $md5Len = 32;
76	+ $this->lockKey = Common::mb_substr($id, 0, self::MAX_KEY_LEN - $md5Len - 1) . md5($id);
77	+ }
78	+
79	+ $lockValue = substr(Common::generateUniqId(), 0, 12);
80	+ $locked = $this->backend->setIfNotExists($this->lockKey, $lockValue, $ttlInSeconds);
81	+
82	+ if ($locked) {
83	+ $this->lockValue = $lockValue;
84	+ }
85	+
86	+ return $locked;
87	+ }
88	+
89	+ public function isLocked()
90	+ {
91	+ if (!$this->lockValue) {
92	+ return false;
93	+ }
94	+
95	+ return $this->lockValue === $this->backend->get($this->lockKey);
96	+ }
97	+
98	+ public function unlock()
99	+ {
100	+ if ($this->lockValue) {
101	+ $this->backend->deleteIfKeyHasValue($this->lockKey, $this->lockValue);
102	+ $this->lockValue = null;
103	+ }
104	+ }
105	+
106	+ public function expireLock($ttlInSeconds)
107	+ {
108	+ if ($ttlInSeconds > 0) {
109	+ if ($this->lockValue) {
110	+ $success = $this->backend->expireIfKeyHasValue($this->lockKey, $this->lockValue, $ttlInSeconds);
111	+ if (!$success) {
112	+ $value = $this->backend->get($this->lockKey);
113	+ $message = sprintf('Failed to expire key %s (%s / %s).', $this->lockKey, $this->lockValue, (string)$value);
114	+
115	+ if ($value === false) {
116	+ Common::printDebug($message . ' It seems like the key already expired as it no longer exists.');
117	+ } elseif (!empty($value) && $value == $this->lockValue) {
118	+ Common::printDebug($message . ' We still have the lock but for some reason it did not expire.');
119	+ } elseif (!empty($value)) {
120	+ Common::printDebug($message . ' This lock has been acquired by another process/server.');
121	+ } else {
122	+ Common::printDebug($message . ' Failed to expire key.');
123	+ }
124	+
125	+ return false;
126	+ }
127	+
128	+ return true;
129	+ } else {
130	+ Common::printDebug('Lock is not acquired, cannot update expiration.');
131	+ }
132	+ } else {
133	+ Common::printDebug('Provided TTL ' . $ttlInSeconds . ' is in valid in Lock::expireLock().');
134	+ }
135	+
136	+ return false;
137	+ }
138	+ }

app/core/Concurrency/LockBackend.php ADDED Viewed

	@@ -0,0 +1,58 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link http://piwik.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+
10	+ namespace Piwik\Concurrency;
11	+
12	+ interface LockBackend
13	+ {
14	+ /**
15	+ * Returns lock keys matching a pattern.
16	+ *
17	+ * @param $pattern
18	+ * @return string[]
19	+ */
20	+ public function getKeysMatchingPattern($pattern);
21	+
22	+ /**
23	+ * Set a key value if the key is not already set.
24	+ *
25	+ * @param $lockKey
26	+ * @param $lockValue
27	+ * @param $ttlInSeconds
28	+ * @return mixed
29	+ */
30	+ public function setIfNotExists($lockKey, $lockValue, $ttlInSeconds);
31	+
32	+ /**
33	+ * Get the lock value for a key if any.
34	+ *
35	+ * @param $lockKey
36	+ * @return mixed
37	+ */
38	+ public function get($lockKey);
39	+
40	+ /**
41	+ * Delete the lock with key = $lockKey if the lock has the given value.
42	+ *
43	+ * @param $lockKey
44	+ * @param $lockValue
45	+ * @return mixed
46	+ */
47	+ public function deleteIfKeyHasValue($lockKey, $lockValue);
48	+
49	+ /**
50	+ * Update expiration for a lock if the lock with the specified key has the given value.
51	+ *
52	+ * @param $lockKey
53	+ * @param $lockValue
54	+ * @param $ttlInSeconds
55	+ * @return mixed
56	+ */
57	+ public function expireIfKeyHasValue($lockKey, $lockValue, $ttlInSeconds);
58	+ }

app/core/Concurrency/LockBackend/MySqlLockBackend.php ADDED Viewed

	@@ -0,0 +1,146 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link http://piwik.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+
10	+ namespace Piwik\Concurrency\LockBackend;
11	+
12	+
13	+ use Piwik\Common;
14	+ use Piwik\Concurrency\LockBackend;
15	+ use Piwik\Db;
16	+ use Piwik\DbHelper;
17	+
18	+ class MySqlLockBackend implements LockBackend
19	+ {
20	+ const TABLE_NAME = 'locks';
21	+
22	+ /**
23	+ * fyi: does not support list keys at the moment just because not really needed so much just yet
24	+ */
25	+ public function getKeysMatchingPattern($pattern)
26	+ {
27	+ $sql = sprintf('SELECT SQL_NO_CACHE distinct `key` FROM %s WHERE `key` like ? and %s', self::getTableName(), $this->getQueryPartExpiryTime());
28	+ $pattern = str_replace('*', '%', $pattern);
29	+ $keys = Db::fetchAll($sql, array($pattern));
30	+ $raw = array_column($keys, 'key');
31	+ return $raw;
32	+ }
33	+
34	+ public function setIfNotExists($key, $value, $ttlInSeconds)
35	+ {
36	+ if (empty($ttlInSeconds)) {
37	+ $ttlInSeconds = 999999999;
38	+ }
39	+
40	+ // FYI: We used to have an INSERT INTO ... ON DUPLICATE UPDATE ... However, this can be problematic in concurrency issues
41	+ // because the ON DUPLICATE UPDATE may work successfully for 2 jobs at the same time but only one of them got the lock then.
42	+ // This would be perfectly fine if we did something like `return $this->get($key) === $value` to 100% detect which process
43	+ // got the lock as we do now. However, maybe the expireTime gets overwritten with a wrong value or so. That's why we
44	+ // rather try to get the lock with the insert only because only one job can succeed with this. If below flow with the
45	+ // delete becomes to slow, we may be able to use the INSERT INTO ... ON DUPLICATE UPDATE again.
46	+
47	+ if ($this->get($key)) {
48	+ return false; // a value is set, won't be possible to insert
49	+ }
50	+
51	+ $tablePrefixed = self::getTableName();
52	+
53	+ // remove any existing but expired lock
54	+ // todo: we could combine get() and keyExists() in one query!
55	+ if ($this->keyExists($key)) {
56	+ // most of the time an expired key should not exist... we don't want to lock the row unncessarily therefore we check first
57	+ // if value exists...
58	+ $sql = sprintf('DELETE FROM %s WHERE `key` = ? and not (%s)', $tablePrefixed, $this->getQueryPartExpiryTime());
59	+ Db::query($sql, array($key));
60	+ }
61	+
62	+ $query = sprintf('INSERT INTO %s (`key`, `value`, `expiry_time`)
63	+ VALUES (?,?,(UNIX_TIMESTAMP() + ?))',
64	+ $tablePrefixed);
65	+ // we make sure to update the row if the key is expired and consider it as "deleted"
66	+
67	+ try {
68	+ Db::query($query, array($key, $value, (int) $ttlInSeconds));
69	+ } catch (\Exception $e) {
70	+ if ($e->getCode() == 23000
71	+ \|\| strpos($e->getMessage(), 'Duplicate entry') !== false
72	+ \|\| strpos($e->getMessage(), ' 1062 ') !== false) {
73	+ return false;
74	+ }
75	+ throw $e;
76	+ }
77	+
78	+ // we make sure we got the lock
79	+ return $this->get($key) === $value;
80	+ }
81	+
82	+ public function get($key)
83	+ {
84	+ $sql = sprintf('SELECT SQL_NO_CACHE `value` FROM %s WHERE `key` = ? AND %s LIMIT 1', self::getTableName(), $this->getQueryPartExpiryTime());
85	+ return Db::fetchOne($sql, array($key));
86	+ }
87	+
88	+ public function deleteIfKeyHasValue($key, $value)
89	+ {
90	+ if (empty($value)) {
91	+ return false;
92	+ }
93	+
94	+ $sql = sprintf('DELETE FROM %s WHERE `key` = ? and `value` = ?', self::getTableName());
95	+ return $this->queryDidMakeChange($sql, array($key, $value));
96	+ }
97	+
98	+ public function expireIfKeyHasValue($key, $value, $ttlInSeconds)
99	+ {
100	+ if (empty($value)) {
101	+ return false;
102	+ }
103	+
104	+ // we need to use unix_timestamp in mysql and not time() in php since the local time might be different on each server
105	+ // better to rely on one central DB server time only
106	+ $sql = sprintf('UPDATE %s SET expiry_time = (UNIX_TIMESTAMP() + ?) WHERE `key` = ? and `value` = ?', self::getTableName());
107	+ $success = $this->queryDidMakeChange($sql, array((int) $ttlInSeconds, $key, $value));
108	+
109	+ if (!$success) {
110	+ // the above update did not work because the same time was already set and we just tried to set the same ttl
111	+ // again too fast within one second
112	+ return $value === $this->get($key);
113	+ }
114	+
115	+ return true;
116	+ }
117	+
118	+ public function keyExists($key)
119	+ {
120	+ $sql = sprintf('SELECT SQL_NO_CACHE 1 FROM %s WHERE `key` = ? LIMIT 1', self::getTableName());
121	+ $value = Db::fetchOne($sql, array($key));
122	+ return !empty($value);
123	+ }
124	+
125	+ private function queryDidMakeChange($sql, $bind = array())
126	+ {
127	+ $query = Db::query($sql, $bind);
128	+ if (is_object($query) && method_exists($query, 'rowCount')) {
129	+ // anything else but mysqli in tracker mode
130	+ return (bool) $query->rowCount();
131	+ } else {
132	+ // mysqli in tracker mode
133	+ return (bool) Db::get()->rowCount($query);
134	+ }
135	+ }
136	+
137	+ private static function getTableName()
138	+ {
139	+ return Common::prefixTable(self::TABLE_NAME);
140	+ }
141	+
142	+ private function getQueryPartExpiryTime()
143	+ {
144	+ return 'UNIX_TIMESTAMP() <= expiry_time';
145	+ }
146	+ }

app/core/Config.php ADDED Viewed

	@@ -0,0 +1,478 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+
10	+ namespace Piwik;
11	+
12	+ use Exception;
13	+ use Piwik\Application\Kernel\GlobalSettingsProvider;
14	+ use Piwik\Config\Cache;
15	+ use Piwik\Config\IniFileChain;
16	+ use Piwik\Container\StaticContainer;
17	+ use Piwik\Exception\MissingFilePermissionException;
18	+ use Piwik\ProfessionalServices\Advertising;
19	+
20	+ /**
21	+ * Singleton that provides read & write access to Piwik's INI configuration.
22	+ *
23	+ * This class reads and writes to the `config/config.ini.php` file. If config
24	+ * options are missing from that file, this class will look for their default
25	+ * values in `config/global.ini.php`.
26	+ *
27	+ * ### Examples
28	+ *
29	+ * Getting a value:
30	+ *
31	+ * // read the minimum_memory_limit option under the [General] section
32	+ * $minValue = Config::getInstance()->General['minimum_memory_limit'];
33	+ *
34	+ * Setting a value:
35	+ *
36	+ * // set the minimum_memory_limit option
37	+ * Config::getInstance()->General['minimum_memory_limit'] = 256;
38	+ * Config::getInstance()->forceSave();
39	+ *
40	+ * Setting an entire section:
41	+ *
42	+ * Config::getInstance()->MySection = array('myoption' => 1);
43	+ * Config::getInstance()->forceSave();
44	+ */
45	+ class Config
46	+ {
47	+ const DEFAULT_LOCAL_CONFIG_PATH = '/config/config.ini.php';
48	+ const DEFAULT_COMMON_CONFIG_PATH = '/config/common.config.ini.php';
49	+ const DEFAULT_GLOBAL_CONFIG_PATH = '/config/global.ini.php';
50	+
51	+ /**
52	+ * @var boolean
53	+ */
54	+ protected $doNotWriteConfigInTests = false;
55	+
56	+ /**
57	+ * @var GlobalSettingsProvider
58	+ */
59	+ protected $settings;
60	+
61	+ /**
62	+ * @return Config
63	+ */
64	+ public static function getInstance()
65	+ {
66	+ return StaticContainer::get('Piwik\Config');
67	+ }
68	+
69	+ public function __construct(GlobalSettingsProvider $settings)
70	+ {
71	+ $this->settings = $settings;
72	+ }
73	+
74	+ /**
75	+ * Returns the path to the local config file used by this instance.
76	+ *
77	+ * @return string
78	+ */
79	+ public function getLocalPath()
80	+ {
81	+ return $this->settings->getPathLocal();
82	+ }
83	+
84	+ /**
85	+ * Returns the path to the global config file used by this instance.
86	+ *
87	+ * @return string
88	+ */
89	+ public function getGlobalPath()
90	+ {
91	+ return $this->settings->getPathGlobal();
92	+ }
93	+
94	+ /**
95	+ * Returns the path to the common config file used by this instance.
96	+ *
97	+ * @return string
98	+ */
99	+ public function getCommonPath()
100	+ {
101	+ return $this->settings->getPathCommon();
102	+ }
103	+
104	+ /**
105	+ * Returns absolute path to the global configuration file
106	+ *
107	+ * @return string
108	+ */
109	+ public static function getGlobalConfigPath()
110	+ {
111	+ return PIWIK_DOCUMENT_ROOT . self::DEFAULT_GLOBAL_CONFIG_PATH;
112	+ }
113	+
114	+ /**
115	+ * Returns absolute path to the common configuration file.
116	+ *
117	+ * @return string
118	+ */
119	+ public static function getCommonConfigPath()
120	+ {
121	+ return PIWIK_USER_PATH . self::DEFAULT_COMMON_CONFIG_PATH;
122	+ }
123	+
124	+ /**
125	+ * Returns default absolute path to the local configuration file.
126	+ *
127	+ * @return string
128	+ */
129	+ public static function getDefaultLocalConfigPath()
130	+ {
131	+ return PIWIK_USER_PATH . self::DEFAULT_LOCAL_CONFIG_PATH;
132	+ }
133	+
134	+ /**
135	+ * Returns absolute path to the local configuration file
136	+ *
137	+ * @return string
138	+ */
139	+ public static function getLocalConfigPath()
140	+ {
141	+ if (!empty($GLOBALS['CONFIG_INI_PATH_RESOLVER']) && is_callable($GLOBALS['CONFIG_INI_PATH_RESOLVER'])) {
142	+ return call_user_func($GLOBALS['CONFIG_INI_PATH_RESOLVER']);
143	+ }
144	+
145	+ $path = self::getByDomainConfigPath();
146	+ if ($path) {
147	+ return $path;
148	+ }
149	+ return self::getDefaultLocalConfigPath();
150	+ }
151	+
152	+ private static function getLocalConfigInfoForHostname($hostname)
153	+ {
154	+ if (!$hostname) {
155	+ return array();
156	+ }
157	+
158	+ // Remove any port number to get actual hostname
159	+ $hostname = Url::getHostSanitized($hostname);
160	+ $standardConfigName = 'config.ini.php';
161	+ $perHostFilename = $hostname . '.' . $standardConfigName;
162	+ $pathDomainConfig = PIWIK_USER_PATH . '/config/' . $perHostFilename;
163	+ $pathDomainMiscUser = PIWIK_USER_PATH . '/misc/user/' . $hostname . '/' . $standardConfigName;
164	+
165	+ $locations = array(
166	+ array('file' => $perHostFilename, 'path' => $pathDomainConfig),
167	+ array('file' => $standardConfigName, 'path' => $pathDomainMiscUser)
168	+ );
169	+
170	+ return $locations;
171	+ }
172	+
173	+ public function getConfigHostnameIfSet()
174	+ {
175	+ if ($this->getByDomainConfigPath() === false) {
176	+ return false;
177	+ }
178	+ return $this->getHostname();
179	+ }
180	+
181	+ public function getClientSideOptions()
182	+ {
183	+ $general = $this->General;
184	+
185	+ return array(
186	+ 'action_url_category_delimiter' => $general['action_url_category_delimiter'],
187	+ 'action_title_category_delimiter' => $general['action_title_category_delimiter'],
188	+ 'autocomplete_min_sites' => $general['autocomplete_min_sites'],
189	+ 'datatable_export_range_as_day' => $general['datatable_export_range_as_day'],
190	+ 'datatable_row_limits' => $this->getDatatableRowLimits(),
191	+ 'are_ads_enabled' => Advertising::isAdsEnabledInConfig($general)
192	+ );
193	+ }
194	+
195	+ /**
196	+ * @param $general
197	+ * @return mixed
198	+ */
199	+ private function getDatatableRowLimits()
200	+ {
201	+ $limits = $this->General['datatable_row_limits'];
202	+ $limits = explode(",", $limits);
203	+ $limits = array_map('trim', $limits);
204	+ return $limits;
205	+ }
206	+
207	+ public static function getByDomainConfigPath()
208	+ {
209	+ $host = self::getHostname();
210	+ $hostConfigs = self::getLocalConfigInfoForHostname($host);
211	+
212	+ foreach ($hostConfigs as $hostConfig) {
213	+ if (Filesystem::isValidFilename($hostConfig['file'])
214	+ && file_exists($hostConfig['path'])
215	+ ) {
216	+ return $hostConfig['path'];
217	+ }
218	+ }
219	+
220	+ return false;
221	+ }
222	+
223	+ /**
224	+ * Returns the hostname of the current request (without port number)
225	+ *
226	+ * @return string
227	+ */
228	+ public static function getHostname()
229	+ {
230	+ // Check trusted requires config file which is not ready yet
231	+ $host = Url::getHost($checkIfTrusted = false);
232	+
233	+ // Remove any port number to get actual hostname
234	+ $host = Url::getHostSanitized($host);
235	+
236	+ return $host;
237	+ }
238	+
239	+ /**
240	+ * If set, Piwik will use the hostname config no matter if it exists or not. Useful for instance if you want to
241	+ * create a new hostname config:
242	+ *
243	+ * $config = Config::getInstance();
244	+ * $config->forceUsageOfHostnameConfig('piwik.example.com');
245	+ * $config->save();
246	+ *
247	+ * @param string $hostname eg piwik.example.com
248	+ * @param string $preferredPath If there are different paths for the config that can be used, eg /config/* and /misc/user/*,
249	+ * and a preferred path is given, then the config path must contain the preferred path.
250	+ * @return string
251	+ * @throws \Exception In case the domain contains not allowed characters
252	+ * @internal
253	+ */
254	+ public function forceUsageOfLocalHostnameConfig($hostname, $preferredPath = null)
255	+ {
256	+ $hostConfigs = self::getLocalConfigInfoForHostname($hostname);
257	+ $fileNames = '';
258	+
259	+ foreach ($hostConfigs as $hostConfig) {
260	+ if (count($hostConfigs) > 1
261	+ && $preferredPath
262	+ && strpos($hostConfig['path'], $preferredPath) === false) {
263	+ continue;
264	+ }
265	+
266	+ $filename = $hostConfig['file'];
267	+ $fileNames .= $filename . ' ';
268	+
269	+ if (Filesystem::isValidFilename($filename)) {
270	+ $pathLocal = $hostConfig['path'];
271	+
272	+ try {
273	+ $this->reload($pathLocal);
274	+ } catch (Exception $ex) {
275	+ // pass (not required for local file to exist at this point)
276	+ }
277	+
278	+ return $pathLocal;
279	+ }
280	+ }
281	+
282	+ throw new Exception('Matomo domain is not a valid looking hostname (' . trim($fileNames) . ').');
283	+ }
284	+
285	+ /**
286	+ * Returns `true` if the local configuration file is writable.
287	+ *
288	+ * @return bool
289	+ */
290	+ public function isFileWritable()
291	+ {
292	+ return is_writable($this->settings->getPathLocal());
293	+ }
294	+
295	+ /**
296	+ * Clear in-memory configuration so it can be reloaded
297	+ * @deprecated since v2.12.0
298	+ */
299	+ public function clear()
300	+ {
301	+ $this->reload();
302	+ }
303	+
304	+ /**
305	+ * Read configuration from files into memory
306	+ *
307	+ * @throws Exception if local config file is not readable; exits for other errors
308	+ * @deprecated since v2.12.0
309	+ */
310	+ public function init()
311	+ {
312	+ $this->reload();
313	+ }
314	+
315	+ /**
316	+ * Reloads config data from disk.
317	+ *
318	+ * @throws \Exception if the global config file is not found and this is a tracker request, or
319	+ * if the local config file is not found and this is NOT a tracker request.
320	+ */
321	+ protected function reload($pathLocal = null, $pathGlobal = null, $pathCommon = null)
322	+ {
323	+ $this->settings->reload($pathGlobal, $pathLocal, $pathCommon);
324	+ }
325	+
326	+ /**
327	+ * @deprecated
328	+ */
329	+ public function existsLocalConfig()
330	+ {
331	+ return is_readable($this->getLocalPath());
332	+ }
333	+
334	+ public function deleteLocalConfig()
335	+ {
336	+ $configLocal = $this->getLocalPath();
337	+
338	+ if(file_exists($configLocal)){
339	+ @unlink($configLocal);
340	+ }
341	+ }
342	+
343	+ /**
344	+ * Returns a configuration value or section by name.
345	+ *
346	+ * @param string $name The value or section name.
347	+ * @return string\|array The requested value requested. Returned by reference.
348	+ * @throws Exception If the value requested not found in either `config.ini.php` or
349	+ * `global.ini.php`.
350	+ * @api
351	+ */
352	+ public function &__get($name)
353	+ {
354	+ $section =& $this->settings->getIniFileChain()->get($name);
355	+ return $section;
356	+ }
357	+
358	+ /**
359	+ * @api
360	+ */
361	+ public function getFromGlobalConfig($name)
362	+ {
363	+ return $this->settings->getIniFileChain()->getFrom($this->getGlobalPath(), $name);
364	+ }
365	+
366	+ /**
367	+ * @api
368	+ */
369	+ public function getFromCommonConfig($name)
370	+ {
371	+ return $this->settings->getIniFileChain()->getFrom($this->getCommonPath(), $name);
372	+ }
373	+
374	+ /**
375	+ * @api
376	+ */
377	+ public function getFromLocalConfig($name)
378	+ {
379	+ return $this->settings->getIniFileChain()->getFrom($this->getLocalPath(), $name);
380	+ }
381	+
382	+ /**
383	+ * Sets a configuration value or section.
384	+ *
385	+ * @param string $name This section name or value name to set.
386	+ * @param mixed $value
387	+ * @api
388	+ */
389	+ public function __set($name, $value)
390	+ {
391	+ $this->settings->getIniFileChain()->set($name, $value);
392	+ }
393	+
394	+ /**
395	+ * Dump config
396	+ *
397	+ * @return string\|null
398	+ * @throws \Exception
399	+ */
400	+ public function dumpConfig()
401	+ {
402	+ $chain = $this->settings->getIniFileChain();
403	+
404	+ $header = "; <?php exit; ?> DO NOT REMOVE THIS LINE\n";
405	+ $header .= "; file automatically generated or modified by Matomo; you can manually override the default values in global.ini.php by redefining them in this file.\n";
406	+ return $chain->dumpChanges($header);
407	+ }
408	+
409	+ /**
410	+ * Write user configuration file
411	+ *
412	+ * @throws \Exception if config file not writable
413	+ */
414	+ protected function writeConfig()
415	+ {
416	+ $output = $this->dumpConfig();
417	+ if ($output !== null
418	+ && $output !== false
419	+ ) {
420	+ $localPath = $this->getLocalPath();
421	+
422	+ if ($this->doNotWriteConfigInTests) {
423	+ // simulate whether it would be successful
424	+ $success = is_writable($localPath);
425	+ } else {
426	+ $success = @file_put_contents($localPath, $output, LOCK_EX);
427	+ }
428	+
429	+ if ($success === false) {
430	+ throw $this->getConfigNotWritableException();
431	+ }
432	+
433	+ $this->settings->getIniFileChain()->deleteConfigCache();
434	+
435	+ /**
436	+ * Triggered when a INI config file is changed on disk.
437	+ *
438	+ * @param string $localPath Absolute path to the changed file on the server.
439	+ */
440	+ Piwik::postEvent('Core.configFileChanged', [$localPath]);
441	+ }
442	+ }
443	+
444	+ /**
445	+ * Writes the current configuration to the config.ini.php file. Only writes options whose
446	+ * values are different from the default.
447	+ *
448	+ * @api
449	+ */
450	+ public function forceSave()
451	+ {
452	+ $this->writeConfig();
453	+ }
454	+
455	+ /**
456	+ * @throws \Exception
457	+ */
458	+ public function getConfigNotWritableException()
459	+ {
460	+ $path = "config/" . basename($this->getLocalPath());
461	+ return new MissingFilePermissionException(Piwik::translate('General_ConfigFileIsNotWritable', array("(" . $path . ")", "")));
462	+ }
463	+
464	+ /**
465	+ * Convenience method for setting settings in a single section. Will set them in a new array first
466	+ * to be compatible with certain PHP versions.
467	+ *
468	+ * @param string $sectionName Section name.
469	+ * @param string $name The setting name.
470	+ * @param mixed $value The setting value to set.
471	+ */
472	+ public static function setSetting($sectionName, $name, $value)
473	+ {
474	+ $section = self::getInstance()->$sectionName;
475	+ $section[$name] = $value;
476	+ self::getInstance()->$sectionName = $section;
477	+ }
478	+ }

app/core/Config/Cache.php ADDED Viewed

	@@ -0,0 +1,89 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link http://piwik.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ */
8	+
9	+ namespace Piwik\Config;
10	+
11	+ use Piwik\Cache\Backend\File;
12	+ use Piwik\Common;
13	+ use Piwik\Filesystem;
14	+ use Piwik\Piwik;
15	+ use Piwik\Url;
16	+
17	+ /**
18	+ * Exception thrown when the config file doesn't exist.
19	+ */
20	+ class Cache extends File
21	+ {
22	+ private $host = '';
23	+
24	+ public function __construct()
25	+ {
26	+ $this->host = $this->getHost();
27	+
28	+ // because the config is not yet loaded we cannot identify the instanceId...
29	+ // need to use the hostname
30	+ $dir = $this->makeCacheDir($this->host);
31	+
32	+ parent::__construct($dir);
33	+ }
34	+
35	+ private function makeCacheDir($host)
36	+ {
37	+ return PIWIK_INCLUDE_PATH . '/tmp/' . $host . '/cache/tracker';
38	+ }
39	+
40	+ public function isValidHost($mergedConfigSettings)
41	+ {
42	+ if (!isset($mergedConfigSettings['General']['trusted_hosts']) \|\| !is_array($mergedConfigSettings['General']['trusted_hosts'])) {
43	+ return false;
44	+ }
45	+ // note: we do not support "enable_trusted_host_check" to keep things secure
46	+ return in_array($this->host, $mergedConfigSettings['General']['trusted_hosts'], true);
47	+ }
48	+
49	+ private function getHost()
50	+ {
51	+ $host = Url::getHost($checkIfTrusted = false);
52	+ $host = Url::getHostSanitized($host); // Remove any port number to get actual hostname
53	+ $host = Common::sanitizeInputValue($host);
54	+
55	+ if (empty($host)
56	+ \|\| strpos($host, '..') !== false
57	+ \|\| strpos($host, '\\') !== false
58	+ \|\| strpos($host, '/') !== false) {
59	+ throw new \Exception('Unsupported host');
60	+ }
61	+
62	+ $this->host = $host;
63	+
64	+ return $host;
65	+ }
66	+
67	+ public function doDelete($id)
68	+ {
69	+ // when the config changes, we need to invalidate the config caches for all configured hosts as well, not only
70	+ // the currently trusted host
71	+ $hosts = Url::getTrustedHosts();
72	+ $initialDir = $this->directory;
73	+
74	+ foreach ($hosts as $host)
75	+ {
76	+ $dir = $this->makeCacheDir($host);
77	+ if (@is_dir($dir)) {
78	+ $this->directory = $dir;
79	+ $success = parent::doDelete($id);
80	+ if ($success) {
81	+ Piwik::postEvent('Core.configFileDeleted', array($this->getFilename($id)));
82	+ }
83	+ }
84	+ }
85	+
86	+ $this->directory = $initialDir;
87	+ }
88	+
89	+ }

app/core/Config/ConfigNotFoundException.php ADDED Viewed

	@@ -0,0 +1,16 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ */
8	+
9	+ namespace Piwik\Config;
10	+
11	+ /**
12	+ * Exception thrown when the config file doesn't exist.
13	+ */
14	+ class ConfigNotFoundException extends \Exception
15	+ {
16	+ }

app/core/Config/IniFileChain.php ADDED Viewed

	@@ -0,0 +1,539 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ */
8	+ namespace Piwik\Config;
9	+
10	+ use Piwik\Common;
11	+ use Piwik\Ini\IniReader;
12	+ use Piwik\Ini\IniReadingException;
13	+ use Piwik\Ini\IniWriter;
14	+ use Piwik\Piwik;
15	+
16	+ /**
17	+ * Manages a list of INI files where the settings in each INI file merge with or override the
18	+ * settings in the previous INI file.
19	+ *
20	+ * The IniFileChain class manages two types of INI files: multiple default setting files and one
21	+ * user settings file.
22	+ *
23	+ * The default setting files (for example, global.ini.php & common.ini.php) hold the default setting values.
24	+ * The settings in these files are merged recursively, however, array settings in one file will still
25	+ * overwrite settings in the previous file.
26	+ *
27	+ * Default settings files cannot be modified through the IniFileChain class.
28	+ *
29	+ * The user settings file (for example, config.ini.php) holds the actual setting values. Settings in the
30	+ * user settings files overwrite other settings. So array settings will not merge w/ previous values.
31	+ *
32	+ * HTML characters and dollar signs are stored as encoded HTML entities in INI files. This prevents
33	+ * several `parse_ini_file` issues, including one where parse_ini_file tries to insert a variable
34	+ * into a setting value if a string like `"$varname" is present.
35	+ */
36	+ class IniFileChain
37	+ {
38	+ const CONFIG_CACHE_KEY = 'config.ini';
39	+ /**
40	+ * Maps INI file names with their parsed contents. The order of the files signifies the order
41	+ * in the chain. Files with lower index are overwritten/merged with files w/ a higher index.
42	+ *
43	+ * @var array
44	+ */
45	+ protected $settingsChain = array();
46	+
47	+ /**
48	+ * The merged INI settings.
49	+ *
50	+ * @var array
51	+ */
52	+ protected $mergedSettings = array();
53	+
54	+ /**
55	+ * Constructor.
56	+ *
57	+ * @param string[] $defaultSettingsFiles The list of paths to INI files w/ the default setting values.
58	+ * @param string\|null $userSettingsFile The path to the user settings file.
59	+ */
60	+ public function __construct(array $defaultSettingsFiles = array(), $userSettingsFile = null)
61	+ {
62	+ $this->reload($defaultSettingsFiles, $userSettingsFile);
63	+ }
64	+
65	+ /**
66	+ * Return setting section by reference.
67	+ *
68	+ * @param string $name
69	+ * @return mixed
70	+ */
71	+ public function &get($name)
72	+ {
73	+ if (!isset($this->mergedSettings[$name])) {
74	+ $this->mergedSettings[$name] = array();
75	+ }
76	+
77	+ $result =& $this->mergedSettings[$name];
78	+ return $result;
79	+ }
80	+
81	+ /**
82	+ * Return setting section from a specific file, rather than the current merged settings.
83	+ *
84	+ * @param string $file The path of the file. Should be the path used in construction or reload().
85	+ * @param string $name The name of the section to access.
86	+ */
87	+ public function getFrom($file, $name)
88	+ {
89	+ return @$this->settingsChain[$file][$name];
90	+ }
91	+
92	+ /**
93	+ * Sets a setting value.
94	+ *
95	+ * @param string $name
96	+ * @param mixed $value
97	+ */
98	+ public function set($name, $value)
99	+ {
100	+ $this->mergedSettings[$name] = $value;
101	+ }
102	+
103	+ /**
104	+ * Returns all settings. Changes made to the array result will be reflected in the
105	+ * IniFileChain instance.
106	+ *
107	+ * @return array
108	+ */
109	+ public function &getAll()
110	+ {
111	+ return $this->mergedSettings;
112	+ }
113	+
114	+ /**
115	+ * Dumps the current in-memory setting values to a string in INI format and returns it.
116	+ *
117	+ * @param string $header The header of the output INI file.
118	+ * @return string The dumped INI contents.
119	+ */
120	+ public function dump($header = '')
121	+ {
122	+ return $this->dumpSettings($this->mergedSettings, $header);
123	+ }
124	+
125	+ /**
126	+ * Writes the difference of the in-memory setting values and the on-disk user settings file setting
127	+ * values to a string in INI format, and returns it.
128	+ *
129	+ * If a config section is identical to the default settings section (as computed by merging
130	+ * all default setting files), it is not written to the user settings file.
131	+ *
132	+ * @param string $header The header of the INI output.
133	+ * @return string The dumped INI contents.
134	+ */
135	+ public function dumpChanges($header = '')
136	+ {
137	+ $userSettingsFile = $this->getUserSettingsFile();
138	+
139	+ $defaultSettings = $this->getMergedDefaultSettings();
140	+ $existingMutableSettings = $this->settingsChain[$userSettingsFile];
141	+
142	+ $dirty = false;
143	+
144	+ $configToWrite = array();
145	+ foreach ($this->mergedSettings as $sectionName => $changedSection) {
146	+ if(isset($existingMutableSettings[$sectionName])){
147	+ $existingMutableSection = $existingMutableSettings[$sectionName];
148	+ } else{
149	+ $existingMutableSection = array();
150	+ }
151	+
152	+ // remove default values from both (they should not get written to local)
153	+ if (isset($defaultSettings[$sectionName])) {
154	+ $changedSection = $this->arrayUnmerge($defaultSettings[$sectionName], $changedSection);
155	+ $existingMutableSection = $this->arrayUnmerge($defaultSettings[$sectionName], $existingMutableSection);
156	+ }
157	+
158	+ // if either local/config have non-default values and the other doesn't,
159	+ // OR both have values, but different values, we must write to config.ini.php
160	+ if (empty($changedSection) xor empty($existingMutableSection)
161	+ \|\| (!empty($changedSection)
162	+ && !empty($existingMutableSection)
163	+ && self::compareElements($changedSection, $existingMutableSection))
164	+ ) {
165	+ $dirty = true;
166	+ }
167	+
168	+ $configToWrite[$sectionName] = $changedSection;
169	+ }
170	+
171	+ if ($dirty) {
172	+ // sort config sections by how early they appear in the file chain
173	+ $self = $this;
174	+ uksort($configToWrite, function ($sectionNameLhs, $sectionNameRhs) use ($self) {
175	+ $lhsIndex = $self->findIndexOfFirstFileWithSection($sectionNameLhs);
176	+ $rhsIndex = $self->findIndexOfFirstFileWithSection($sectionNameRhs);
177	+
178	+ if ($lhsIndex == $rhsIndex) {
179	+ $lhsIndexInFile = $self->getIndexOfSectionInFile($lhsIndex, $sectionNameLhs);
180	+ $rhsIndexInFile = $self->getIndexOfSectionInFile($rhsIndex, $sectionNameRhs);
181	+
182	+ if ($lhsIndexInFile == $rhsIndexInFile) {
183	+ return 0;
184	+ } elseif ($lhsIndexInFile < $rhsIndexInFile) {
185	+ return -1;
186	+ } else {
187	+ return 1;
188	+ }
189	+ } elseif ($lhsIndex < $rhsIndex) {
190	+ return -1;
191	+ } else {
192	+ return 1;
193	+ }
194	+ });
195	+
196	+ return $this->dumpSettings($configToWrite, $header);
197	+ } else {
198	+ return null;
199	+ }
200	+ }
201	+
202	+ /**
203	+ * Reloads settings from disk.
204	+ */
205	+ public function reload($defaultSettingsFiles = array(), $userSettingsFile = null)
206	+ {
207	+ if (!empty($defaultSettingsFiles)
208	+ \|\| !empty($userSettingsFile)
209	+ ) {
210	+ $this->resetSettingsChain($defaultSettingsFiles, $userSettingsFile);
211	+ }
212	+
213	+ if (!empty($userSettingsFile) && !empty($GLOBALS['ENABLE_CONFIG_PHP_CACHE'])) {
214	+ $cache = new Cache();
215	+ $values = $cache->doFetch(self::CONFIG_CACHE_KEY);
216	+
217	+ if (!empty($values)
218	+ && isset($values['mergedSettings'])
219	+ && isset($values['settingsChain'])) {
220	+ $this->mergedSettings = $values['mergedSettings'];
221	+ $this->settingsChain = $values['settingsChain'];
222	+ return;
223	+ }
224	+ }
225	+
226	+ $reader = new IniReader();
227	+ foreach ($this->settingsChain as $file => $ignore) {
228	+ if (is_readable($file)) {
229	+ try {
230	+ $contents = $reader->readFile($file);
231	+ $this->settingsChain[$file] = $this->decodeValues($contents);
232	+ } catch (IniReadingException $ex) {
233	+ throw new IniReadingException('Unable to read INI file {' . $file . '}: ' . $ex->getMessage() . "\n Your host may have disabled parse_ini_file().");
234	+ }
235	+
236	+ $this->decodeValues($this->settingsChain[$file]);
237	+ }
238	+ }
239	+
240	+ $merged = $this->mergeFileSettings();
241	+ // remove reference to $this->settingsChain... otherwise dump() or compareElements() will never notice a difference
242	+ // on PHP 7+ as they would be always equal
243	+ $this->mergedSettings = $this->copy($merged);
244	+
245	+ if (!empty($GLOBALS['MATOMO_MODIFY_CONFIG_SETTINGS']) && !empty($this->mergedSettings)) {
246	+ $this->mergedSettings = call_user_func($GLOBALS['MATOMO_MODIFY_CONFIG_SETTINGS'], $this->mergedSettings);
247	+ }
248	+
249	+ if (!empty($GLOBALS['ENABLE_CONFIG_PHP_CACHE'])
250	+ && !empty($userSettingsFile)
251	+ && !empty($this->mergedSettings)
252	+ && !empty($this->settingsChain)) {
253	+
254	+ $ttlOneHour = 3600;
255	+ $cache = new Cache();
256	+ if ($cache->isValidHost($this->mergedSettings)) {
257	+ // we make sure to save the config only if the host is valid...
258	+ $data = array('mergedSettings' => $this->mergedSettings, 'settingsChain' => $this->settingsChain);
259	+ $cache->doSave(self::CONFIG_CACHE_KEY, $data, $ttlOneHour);
260	+ }
261	+ }
262	+ }
263	+
264	+ public function deleteConfigCache()
265	+ {
266	+ if (!empty($GLOBALS['ENABLE_CONFIG_PHP_CACHE'])) {
267	+ $cache = new Cache();
268	+ $cache->doDelete(IniFileChain::CONFIG_CACHE_KEY);
269	+ }
270	+ }
271	+
272	+ private function copy($merged)
273	+ {
274	+ $copy = array();
275	+ foreach ($merged as $index => $value) {
276	+ if (is_array($value)) {
277	+ $copy[$index] = $this->copy($value);
278	+ } else {
279	+ $copy[$index] = $value;
280	+ }
281	+ }
282	+ return $copy;
283	+ }
284	+
285	+ private function resetSettingsChain($defaultSettingsFiles, $userSettingsFile)
286	+ {
287	+ $this->settingsChain = array();
288	+
289	+ if (!empty($defaultSettingsFiles)) {
290	+ foreach ($defaultSettingsFiles as $file) {
291	+ $this->settingsChain[$file] = null;
292	+ }
293	+ }
294	+
295	+ if (!empty($userSettingsFile)) {
296	+ $this->settingsChain[$userSettingsFile] = null;
297	+ }
298	+ }
299	+
300	+ protected function mergeFileSettings()
301	+ {
302	+ $mergedSettings = $this->getMergedDefaultSettings();
303	+
304	+ $userSettings = end($this->settingsChain) ?: array();
305	+ foreach ($userSettings as $sectionName => $section) {
306	+ if (!isset($mergedSettings[$sectionName])) {
307	+ $mergedSettings[$sectionName] = $section;
308	+ } else {
309	+ // the last user settings file completely overwrites INI sections. the other files in the chain
310	+ // can add to array options
311	+ $mergedSettings[$sectionName] = array_merge($mergedSettings[$sectionName], $section);
312	+ }
313	+ }
314	+
315	+ return $mergedSettings;
316	+ }
317	+
318	+ protected function getMergedDefaultSettings()
319	+ {
320	+ $userSettingsFile = $this->getUserSettingsFile();
321	+
322	+ $mergedSettings = array();
323	+ foreach ($this->settingsChain as $file => $settings) {
324	+ if ($file == $userSettingsFile
325	+ \|\| empty($settings)
326	+ ) {
327	+ continue;
328	+ }
329	+
330	+ foreach ($settings as $sectionName => $section) {
331	+ if (!isset($mergedSettings[$sectionName])) {
332	+ $mergedSettings[$sectionName] = $section;
333	+ } else {
334	+ $mergedSettings[$sectionName] = $this->array_merge_recursive_distinct($mergedSettings[$sectionName], $section);
335	+ }
336	+ }
337	+ }
338	+ return $mergedSettings;
339	+ }
340	+
341	+ protected function getUserSettingsFile()
342	+ {
343	+ // the user settings file is the last key in $settingsChain
344	+ end($this->settingsChain);
345	+ return key($this->settingsChain);
346	+ }
347	+
348	+ /**
349	+ * Comparison function
350	+ *
351	+ * @param mixed $elem1
352	+ * @param mixed $elem2
353	+ * @return int;
354	+ */
355	+ public static function compareElements($elem1, $elem2)
356	+ {
357	+ if (is_array($elem1)) {
358	+ if (is_array($elem2)) {
359	+ return strcmp(serialize($elem1), serialize($elem2));
360	+ }
361	+
362	+ return 1;
363	+ }
364	+
365	+ if (is_array($elem2)) {
366	+ return -1;
367	+ }
368	+
369	+ if ((string)$elem1 === (string)$elem2) {
370	+ return 0;
371	+ }
372	+
373	+ return ((string)$elem1 > (string)$elem2) ? 1 : -1;
374	+ }
375	+
376	+ /**
377	+ * Compare arrays and return difference, such that:
378	+ *
379	+ * $modified = array_merge($original, $difference);
380	+ *
381	+ * @param array $original original array
382	+ * @param array $modified modified array
383	+ * @return array differences between original and modified
384	+ */
385	+ public function arrayUnmerge($original, $modified)
386	+ {
387	+ // return key/value pairs for keys in $modified but not in $original
388	+ // return key/value pairs for keys in both $modified and $original, but values differ
389	+ // ignore keys that are in $original but not in $modified
390	+
391	+ if (empty($original) \|\| !is_array($original)) {
392	+ $original = array();
393	+ }
394	+
395	+ if (empty($modified) \|\| !is_array($modified)) {
396	+ $modified = array();
397	+ }
398	+
399	+ return array_udiff_assoc($modified, $original, array(__CLASS__, 'compareElements'));
400	+ }
401	+
402	+ /**
403	+ * array_merge_recursive does indeed merge arrays, but it converts values with duplicate
404	+ * keys to arrays rather than overwriting the value in the first array with the duplicate
405	+ * value in the second array, as array_merge does. I.e., with array_merge_recursive,
406	+ * this happens (documented behavior):
407	+ *
408	+ * array_merge_recursive(array('key' => 'org value'), array('key' => 'new value'));
409	+ * => array('key' => array('org value', 'new value'));
410	+ *
411	+ * array_merge_recursive_distinct does not change the datatypes of the values in the arrays.
412	+ * Matching keys' values in the second array overwrite those in the first array, as is the
413	+ * case with array_merge, i.e.:
414	+ *
415	+ * array_merge_recursive_distinct(array('key' => 'org value'), array('key' => 'new value'));
416	+ * => array('key' => array('new value'));
417	+ *
418	+ * Parameters are passed by reference, though only for performance reasons. They're not
419	+ * altered by this function.
420	+ *
421	+ * @param array $array1
422	+ * @param array $array2
423	+ * @return array
424	+ * @author Daniel <daniel (at) danielsmedegaardbuus (dot) dk>
425	+ * @author Gabriel Sobrinho <gabriel (dot) sobrinho (at) gmail (dot) com>
426	+ */
427	+ private function array_merge_recursive_distinct(array &$array1, array &$array2)
428	+ {
429	+ $merged = $array1;
430	+ foreach ($array2 as $key => &$value) {
431	+ if (is_array($value) && isset($merged [$key]) && is_array($merged [$key])) {
432	+ $merged [$key] = $this->array_merge_recursive_distinct($merged [$key], $value);
433	+ } else {
434	+ $merged [$key] = $value;
435	+ }
436	+ }
437	+ return $merged;
438	+ }
439	+
440	+ /**
441	+ * public for use in closure.
442	+ */
443	+ public function findIndexOfFirstFileWithSection($sectionName)
444	+ {
445	+ $count = 0;
446	+ foreach ($this->settingsChain as $file => $settings) {
447	+ if (isset($settings[$sectionName])) {
448	+ break;
449	+ }
450	+
451	+ ++$count;
452	+ }
453	+ return $count;
454	+ }
455	+
456	+ /**
457	+ * public for use in closure.
458	+ */
459	+ public function getIndexOfSectionInFile($fileIndex, $sectionName)
460	+ {
461	+ reset($this->settingsChain);
462	+ for ($i = 0; $i != $fileIndex; ++$i) {
463	+ next($this->settingsChain);
464	+ }
465	+
466	+ $settingsData = current($this->settingsChain);
467	+ if (empty($settingsData)) {
468	+ return -1;
469	+ }
470	+
471	+ $settingsDataSectionNames = array_keys($settingsData);
472	+
473	+ return array_search($sectionName, $settingsDataSectionNames);
474	+ }
475	+
476	+ /**
477	+ * Encode HTML entities
478	+ *
479	+ * @param mixed $values
480	+ * @return mixed
481	+ */
482	+ protected function encodeValues(&$values)
483	+ {
484	+ if (is_array($values)) {
485	+ foreach ($values as &$value) {
486	+ $value = $this->encodeValues($value);
487	+ }
488	+ } elseif (is_float($values)) {
489	+ $values = Common::forceDotAsSeparatorForDecimalPoint($values);
490	+ } elseif (is_string($values)) {
491	+ $values = htmlentities($values, ENT_COMPAT, 'UTF-8');
492	+ $values = str_replace('$', '$', $values);
493	+ }
494	+ return $values;
495	+ }
496	+
497	+ /**
498	+ * Decode HTML entities
499	+ *
500	+ * @param mixed $values
501	+ * @return mixed
502	+ */
503	+ protected function decodeValues(&$values)
504	+ {
505	+ if (is_array($values)) {
506	+ foreach ($values as &$value) {
507	+ $value = $this->decodeValues($value);
508	+ }
509	+ return $values;
510	+ } elseif (is_string($values)) {
511	+ return html_entity_decode($values, ENT_COMPAT, 'UTF-8');
512	+ }
513	+ return $values;
514	+ }
515	+
516	+ private function dumpSettings($values, $header)
517	+ {
518	+ /**
519	+ * Triggered before a config is being written / saved on the local file system.
520	+ *
521	+ * A plugin can listen to it and modify which settings will be saved on the file system. This allows you
522	+ * to prevent saving config values that a plugin sets on demand. Say you configure the database password in the
523	+ * config on demand in your plugin, then you could prevent that the password is saved in the actual config file
524	+ * by listening to this event like this:
525	+ *
526	+ * Example
527	+ * function doNotSaveDbPassword (&$values) {
528	+ * unset($values['database']['password']);
529	+ * }
530	+ *
531	+ * @param array &$values Config values that will be saved
532	+ */
533	+ Piwik::postEvent('Config.beforeSave', array(&$values));
534	+ $values = $this->encodeValues($values);
535	+
536	+ $writer = new IniWriter();
537	+ return $writer->writeToString($values, $header);
538	+ }
539	+ }

app/core/Console.php ADDED Viewed

	@@ -0,0 +1,319 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik;
10	+
11	+ use Exception;
12	+ use Monolog\Handler\FingersCrossedHandler;
13	+ use Piwik\Application\Environment;
14	+ use Piwik\Config\ConfigNotFoundException;
15	+ use Piwik\Container\StaticContainer;
16	+ use Piwik\Exception\AuthenticationFailedException;
17	+ use Piwik\Plugin\Manager as PluginManager;
18	+ use Piwik\Plugins\Monolog\Handler\FailureLogMessageDetector;
19	+ use Piwik\Version;
20	+ use Psr\Log\LoggerInterface;
21	+ use Symfony\Bridge\Monolog\Handler\ConsoleHandler;
22	+ use Symfony\Component\Console\Application;
23	+ use Symfony\Component\Console\Command\Command;
24	+ use Symfony\Component\Console\Input\InputInterface;
25	+ use Symfony\Component\Console\Input\InputOption;
26	+ use Symfony\Component\Console\Output\OutputInterface;
27	+
28	+ class Console extends Application
29	+ {
30	+ /**
31	+ * @var Environment
32	+ */
33	+ private $environment;
34	+
35	+ public function __construct(Environment $environment = null)
36	+ {
37	+ $this->setServerArgsIfPhpCgi();
38	+
39	+ parent::__construct('Matomo', Version::VERSION);
40	+
41	+ $this->environment = $environment;
42	+
43	+ $option = new InputOption('matomo-domain',
44	+ null,
45	+ InputOption::VALUE_OPTIONAL,
46	+ 'Matomo URL (protocol and domain) eg. "http://matomo.example.org"'
47	+ );
48	+
49	+ $this->getDefinition()->addOption($option);
50	+
51	+ // @todo Remove this alias in Matomo 4.0
52	+ $option = new InputOption('piwik-domain',
53	+ null,
54	+ InputOption::VALUE_OPTIONAL,
55	+ '[DEPRECATED] Matomo URL (protocol and domain) eg. "http://matomo.example.org"'
56	+ );
57	+
58	+ $this->getDefinition()->addOption($option);
59	+
60	+ $option = new InputOption('xhprof',
61	+ null,
62	+ InputOption::VALUE_NONE,
63	+ 'Enable profiling with XHProf'
64	+ );
65	+
66	+ $this->getDefinition()->addOption($option);
67	+ }
68	+
69	+ public function renderException($e, $output)
70	+ {
71	+ $logHandlers = StaticContainer::get('log.handlers');
72	+
73	+ $hasFingersCrossed = false;
74	+ foreach ($logHandlers as $handler) {
75	+ if ($handler instanceof FingersCrossedHandler) {
76	+ $hasFingersCrossed = true;
77	+ continue;
78	+ }
79	+ }
80	+
81	+ if ($hasFingersCrossed
82	+ && $output->getVerbosity() < OutputInterface::VERBOSITY_VERBOSE
83	+ ) {
84	+ $output->setVerbosity(OutputInterface::VERBOSITY_VERBOSE);
85	+ }
86	+
87	+ parent::renderException($e, $output);
88	+ }
89	+
90	+ public function doRun(InputInterface $input, OutputInterface $output)
91	+ {
92	+ try {
93	+ return $this->doRunImpl($input, $output);
94	+ } catch (\Exception $ex) {
95	+ try {
96	+ FrontController::generateSafeModeOutputFromException($ex);
97	+ } catch (\Exception $ex) {
98	+ // ignore, we re-throw the original exception, not a wrapped one
99	+ }
100	+
101	+ throw $ex;
102	+ }
103	+ }
104	+
105	+ private function doRunImpl(InputInterface $input, OutputInterface $output)
106	+ {
107	+ if ($input->hasParameterOption('--xhprof')) {
108	+ Profiler::setupProfilerXHProf(true, true);
109	+ }
110	+
111	+ $this->initMatomoHost($input);
112	+ $this->initEnvironment($output);
113	+ $this->initLoggerOutput($output);
114	+
115	+ try {
116	+ self::initPlugins();
117	+ } catch (ConfigNotFoundException $e) {
118	+ // Piwik not installed yet, no config file?
119	+ Log::warning($e->getMessage());
120	+ }
121	+
122	+ $this->initAuth();
123	+
124	+ $commands = $this->getAvailableCommands();
125	+
126	+ foreach ($commands as $command) {
127	+ $this->addCommandIfExists($command);
128	+ }
129	+
130	+ $exitCode = null;
131	+
132	+ /**
133	+ * @ignore
134	+ */
135	+ Piwik::postEvent('Console.doRun', [&$exitCode, $input, $output]);
136	+
137	+ if ($exitCode === null) {
138	+ $self = $this;
139	+ $exitCode = Access::doAsSuperUser(function () use ($input, $output, $self) {
140	+ return call_user_func(array($self, 'Symfony\Component\Console\Application::doRun'), $input, $output);
141	+ });
142	+ }
143	+
144	+ $importantLogDetector = StaticContainer::get(FailureLogMessageDetector::class);
145	+ if ($exitCode === 0 && $importantLogDetector->hasEncounteredImportantLog()) {
146	+ $output->writeln("Error: error or warning logs detected, exit 1");
147	+ $exitCode = 1;
148	+ }
149	+
150	+ return $exitCode;
151	+ }
152	+
153	+ private function addCommandIfExists($command)
154	+ {
155	+ if (!class_exists($command)) {
156	+ Log::warning(sprintf('Cannot add command %s, class does not exist', $command));
157	+ } elseif (!is_subclass_of($command, 'Piwik\Plugin\ConsoleCommand')) {
158	+ Log::warning(sprintf('Cannot add command %s, class does not extend Piwik\Plugin\ConsoleCommand', $command));
159	+ } else {
160	+ /** @var Command $commandInstance */
161	+ $commandInstance = new $command;
162	+
163	+ // do not add the command if it already exists; this way we can add the command ourselves in tests
164	+ if (!$this->has($commandInstance->getName())) {
165	+ $this->add($commandInstance);
166	+ }
167	+ }
168	+ }
169	+
170	+ /**
171	+ * Returns a list of available command classnames.
172	+ *
173	+ * @return string[]
174	+ */
175	+ private function getAvailableCommands()
176	+ {
177	+ $commands = $this->getDefaultPiwikCommands();
178	+ $detected = PluginManager::getInstance()->findMultipleComponents('Commands', 'Piwik\\Plugin\\ConsoleCommand');
179	+
180	+ $commands = array_merge($commands, $detected);
181	+
182	+ /**
183	+ * Triggered to filter / restrict console commands. Plugins that want to restrict commands
184	+ * should subscribe to this event and remove commands from the existing list.
185	+ *
186	+ * Example
187	+ *
188	+ * public function filterConsoleCommands(&$commands)
189	+ * {
190	+ * $key = array_search('Piwik\Plugins\MyPlugin\Commands\MyCommand', $commands);
191	+ * if (false !== $key) {
192	+ * unset($commands[$key]);
193	+ * }
194	+ * }
195	+ *
196	+ * @param array &$commands An array containing a list of command class names.
197	+ */
198	+ Piwik::postEvent('Console.filterCommands', array(&$commands));
199	+
200	+ $commands = array_values(array_unique($commands));
201	+
202	+ return $commands;
203	+ }
204	+
205	+ private function setServerArgsIfPhpCgi()
206	+ {
207	+ if (Common::isPhpCgiType()) {
208	+ $_SERVER['argv'] = array();
209	+ foreach ($_GET as $name => $value) {
210	+ $argument = $name;
211	+ if (!empty($value)) {
212	+ $argument .= '=' . $value;
213	+ }
214	+
215	+ $_SERVER['argv'][] = $argument;
216	+ }
217	+
218	+ if (!defined('STDIN')) {
219	+ define('STDIN', fopen('php://stdin', 'r'));
220	+ }
221	+ }
222	+ }
223	+
224	+ public static function isSupported()
225	+ {
226	+ return Common::isPhpCliMode() && !Common::isPhpCgiType();
227	+ }
228	+
229	+ protected function initMatomoHost(InputInterface $input)
230	+ {
231	+ $matomoHostname = $input->getParameterOption('--matomo-domain');
232	+
233	+ if (empty($matomoHostname)) {
234	+ $matomoHostname = $input->getParameterOption('--piwik-domain');
235	+ }
236	+
237	+ if (empty($matomoHostname)) {
238	+ $matomoHostname = $input->getParameterOption('--url');
239	+ }
240	+
241	+ $matomoHostname = UrlHelper::getHostFromUrl($matomoHostname);
242	+ Url::setHost($matomoHostname);
243	+ }
244	+
245	+ protected function initEnvironment(OutputInterface $output)
246	+ {
247	+ try {
248	+ if ($this->environment === null) {
249	+ $this->environment = new Environment('cli');
250	+ $this->environment->init();
251	+ }
252	+
253	+ $config = Config::getInstance();
254	+ return $config;
255	+ } catch (\Exception $e) {
256	+ $output->writeln($e->getMessage() . "\n");
257	+ }
258	+ }
259	+
260	+ /**
261	+ * Register the console output into the logger.
262	+ *
263	+ * Ideally, this should be done automatically with events:
264	+ * @see http://symfony.com/fr/doc/current/components/console/events.html
265	+ * @see Symfony\Bridge\Monolog\Handler\ConsoleHandler::onCommand()
266	+ * But it would require to install Symfony's Event Dispatcher.
267	+ */
268	+ private function initLoggerOutput(OutputInterface $output)
269	+ {
270	+ /** @var ConsoleHandler $consoleLogHandler */
271	+ $consoleLogHandler = StaticContainer::get('Symfony\Bridge\Monolog\Handler\ConsoleHandler');
272	+ $consoleLogHandler->setOutput($output);
273	+ }
274	+
275	+ public static function initPlugins()
276	+ {
277	+ Plugin\Manager::getInstance()->loadActivatedPlugins();
278	+ Plugin\Manager::getInstance()->loadPluginTranslations();
279	+ }
280	+
281	+ private function getDefaultPiwikCommands()
282	+ {
283	+ $commands = array(
284	+ 'Piwik\CliMulti\RequestCommand'
285	+ );
286	+
287	+ $commandsFromPluginsMarkedInConfig = $this->getCommandsFromPluginsMarkedInConfig();
288	+ $commands = array_merge($commands, $commandsFromPluginsMarkedInConfig);
289	+
290	+ return $commands;
291	+ }
292	+
293	+ private function getCommandsFromPluginsMarkedInConfig()
294	+ {
295	+ $plugins = Config::getInstance()->General['always_load_commands_from_plugin'];
296	+ $plugins = explode(',', $plugins);
297	+
298	+ $commands = array();
299	+ foreach($plugins as $plugin) {
300	+ $instance = new Plugin($plugin);
301	+ $commands = array_merge($commands, $instance->findMultipleComponents('Commands', 'Piwik\\Plugin\\ConsoleCommand'));
302	+ }
303	+ return $commands;
304	+ }
305	+
306	+ private function initAuth()
307	+ {
308	+ Piwik::postEvent('Request.initAuthenticationObject');
309	+ try {
310	+ StaticContainer::get('Piwik\Auth');
311	+ } catch (Exception $e) {
312	+ $message = "Authentication object cannot be found in the container. Maybe the Login plugin is not activated?
313	+ You can activate the plugin by adding:
314	+ Plugins[] = Login
315	+ under the [Plugins] section in your config/config.ini.php";
316	+ StaticContainer::get(LoggerInterface::class)->warning($message);
317	+ }
318	+ }
319	+ }

app/core/Container/ContainerDoesNotExistException.php ADDED Viewed

	@@ -0,0 +1,18 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ */
8	+
9	+ namespace Piwik\Container;
10	+
11	+ use RuntimeException;
12	+
13	+ /**
14	+ * Thrown if the root container has not been created and set in StaticContainer.
15	+ */
16	+ class ContainerDoesNotExistException extends RuntimeException
17	+ {
18	+ }

app/core/Container/ContainerFactory.php ADDED Viewed

	@@ -0,0 +1,152 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ */
8	+
9	+ namespace Piwik\Container;
10	+
11	+ use DI\Container;
12	+ use DI\ContainerBuilder;
13	+ use Doctrine\Common\Cache\ArrayCache;
14	+ use Piwik\Application\Kernel\GlobalSettingsProvider;
15	+ use Piwik\Application\Kernel\PluginList;
16	+ use Piwik\Plugin\Manager;
17	+
18	+ /**
19	+ * Creates a configured DI container.
20	+ */
21	+ class ContainerFactory
22	+ {
23	+ /**
24	+ * @var PluginList
25	+ */
26	+ private $pluginList;
27	+
28	+ /**
29	+ * @var GlobalSettingsProvider
30	+ */
31	+ private $settings;
32	+
33	+ /**
34	+ * Optional environment configs to load.
35	+ *
36	+ * @var string[]
37	+ */
38	+ private $environments;
39	+
40	+ /**
41	+ * @var array[]
42	+ */
43	+ private $definitions;
44	+
45	+ /**
46	+ * @param PluginList $pluginList
47	+ * @param GlobalSettingsProvider $settings
48	+ * @param string[] $environment Optional environment configs to load.
49	+ * @param array[] $definitions
50	+ */
51	+ public function __construct(PluginList $pluginList, GlobalSettingsProvider $settings, array $environments = array(), array $definitions = array())
52	+ {
53	+ $this->pluginList = $pluginList;
54	+ $this->settings = $settings;
55	+ $this->environments = $environments;
56	+ $this->definitions = $definitions;
57	+ }
58	+
59	+ /**
60	+ * @link http://php-di.org/doc/container-configuration.html
61	+ * @throws \Exception
62	+ * @return Container
63	+ */
64	+ public function create()
65	+ {
66	+ $builder = new ContainerBuilder();
67	+
68	+ $builder->useAnnotations(false);
69	+ $builder->setDefinitionCache(new ArrayCache());
70	+
71	+ // INI config
72	+ $builder->addDefinitions(new IniConfigDefinitionSource($this->settings));
73	+
74	+ // Global config
75	+ $builder->addDefinitions(PIWIK_DOCUMENT_ROOT . '/config/global.php');
76	+
77	+ // Plugin configs
78	+ $this->addPluginConfigs($builder);
79	+
80	+ // Development config
81	+ if ($this->isDevelopmentModeEnabled()) {
82	+ $this->addEnvironmentConfig($builder, 'dev');
83	+ }
84	+
85	+ // Environment config
86	+ foreach ($this->environments as $environment) {
87	+ $this->addEnvironmentConfig($builder, $environment);
88	+ }
89	+
90	+ // User config
91	+ if (file_exists(PIWIK_USER_PATH . '/config/config.php')
92	+ && !in_array('test', $this->environments, true)) {
93	+ $builder->addDefinitions(PIWIK_USER_PATH . '/config/config.php');
94	+ }
95	+
96	+ if (!empty($this->definitions)) {
97	+ foreach ($this->definitions as $definitionArray) {
98	+ $builder->addDefinitions($definitionArray);
99	+ }
100	+ }
101	+
102	+ $container = $builder->build();
103	+ $container->set('Piwik\Application\Kernel\PluginList', $this->pluginList);
104	+ $container->set('Piwik\Application\Kernel\GlobalSettingsProvider', $this->settings);
105	+
106	+ return $container;
107	+ }
108	+
109	+ private function addEnvironmentConfig(ContainerBuilder $builder, $environment)
110	+ {
111	+ if (!$environment) {
112	+ return;
113	+ }
114	+
115	+ $file = sprintf('%s/config/environment/%s.php', PIWIK_USER_PATH, $environment);
116	+
117	+ if (file_exists($file)) {
118	+ $builder->addDefinitions($file);
119	+ }
120	+
121	+ // add plugin environment configs
122	+ $plugins = $this->pluginList->getActivatedPlugins();
123	+ foreach ($plugins as $plugin) {
124	+ $baseDir = Manager::getPluginDirectory($plugin);
125	+
126	+ $environmentFile = $baseDir . '/config/' . $environment . '.php';
127	+ if (file_exists($environmentFile)) {
128	+ $builder->addDefinitions($environmentFile);
129	+ }
130	+ }
131	+ }
132	+
133	+ private function addPluginConfigs(ContainerBuilder $builder)
134	+ {
135	+ $plugins = $this->pluginList->getActivatedPlugins();
136	+
137	+ foreach ($plugins as $plugin) {
138	+ $baseDir = Manager::getPluginDirectory($plugin);
139	+
140	+ $file = $baseDir . '/config/config.php';
141	+ if (file_exists($file)) {
142	+ $builder->addDefinitions($file);
143	+ }
144	+ }
145	+ }
146	+
147	+ private function isDevelopmentModeEnabled()
148	+ {
149	+ $section = $this->settings->getSection('Development');
150	+ return (bool) @$section['enabled']; // TODO: code redundancy w/ Development. hopefully ok for now.
151	+ }
152	+ }

app/core/Container/IniConfigDefinitionSource.php ADDED Viewed

	@@ -0,0 +1,95 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ */
8	+
9	+ namespace Piwik\Container;
10	+
11	+ use DI\Definition\Exception\DefinitionException;
12	+ use DI\Definition\Source\DefinitionSource;
13	+ use DI\Definition\ValueDefinition;
14	+ use Piwik\Application\Kernel\GlobalSettingsProvider;
15	+
16	+ /**
17	+ * Expose the INI config into PHP-DI.
18	+ *
19	+ * The INI config can be used by prefixing `ini.` before the setting we want to get:
20	+ *
21	+ * $maintenanceMode = $container->get('ini.General.maintenance_mode');
22	+ */
23	+ class IniConfigDefinitionSource implements DefinitionSource
24	+ {
25	+ /**
26	+ * @var GlobalSettingsProvider
27	+ */
28	+ private $config;
29	+
30	+ /**
31	+ * @var string
32	+ */
33	+ private $prefix;
34	+
35	+ /**
36	+ * @param GlobalSettingsProvider $config
37	+ * @param string $prefix Prefix for the container entries.
38	+ */
39	+ public function __construct(GlobalSettingsProvider $config, $prefix = 'ini.')
40	+ {
41	+ $this->config = $config;
42	+ $this->prefix = $prefix;
43	+ }
44	+
45	+ /**
46	+ * {@inheritdoc}
47	+ */
48	+ public function getDefinition($name)
49	+ {
50	+ if (strpos($name, $this->prefix) !== 0) {
51	+ return null;
52	+ }
53	+
54	+ list($sectionName, $configKey) = $this->parseEntryName($name);
55	+
56	+ $section = $this->getSection($sectionName);
57	+
58	+ if ($configKey === null) {
59	+ return new ValueDefinition($name, $section);
60	+ }
61	+
62	+ if (! array_key_exists($configKey, $section)) {
63	+ return null;
64	+ }
65	+
66	+ return new ValueDefinition($name, $section[$configKey]);
67	+ }
68	+
69	+ private function parseEntryName($name)
70	+ {
71	+ $parts = explode('.', $name, 3);
72	+
73	+ array_shift($parts);
74	+
75	+ if (! isset($parts[1])) {
76	+ $parts[1] = null;
77	+ }
78	+
79	+ return $parts;
80	+ }
81	+
82	+ private function getSection($sectionName)
83	+ {
84	+ $section = $this->config->getSection($sectionName);
85	+
86	+ if (!is_array($section)) {
87	+ throw new DefinitionException(sprintf(
88	+ 'IniFileChain did not return an array for the config section %s',
89	+ $section
90	+ ));
91	+ }
92	+
93	+ return $section;
94	+ }
95	+ }

app/core/Container/StaticContainer.php ADDED Viewed

	@@ -0,0 +1,87 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ */
8	+
9	+ namespace Piwik\Container;
10	+
11	+ use DI\Container;
12	+
13	+ /**
14	+ * This class provides a static access to the container.
15	+ *
16	+ * @deprecated This class is introduced only to keep BC with the current static architecture. It will be removed in 3.0.
17	+ * - it is global state (that class makes the container a global variable)
18	+ * - using the container directly is the "service locator" anti-pattern (which is not dependency injection)
19	+ */
20	+ class StaticContainer
21	+ {
22	+ /**
23	+ * @var Container[]
24	+ */
25	+ private static $containerStack = array();
26	+
27	+ /**
28	+ * Definitions to register in the container.
29	+ *
30	+ * @var array[]
31	+ */
32	+ private static $definitions = array();
33	+
34	+ /**
35	+ * @return Container
36	+ */
37	+ public static function getContainer()
38	+ {
39	+ if (empty(self::$containerStack)) {
40	+ throw new ContainerDoesNotExistException("The root container has not been created yet.");
41	+ }
42	+
43	+ return end(self::$containerStack);
44	+ }
45	+
46	+ public static function clearContainer()
47	+ {
48	+ self::pop();
49	+ }
50	+
51	+ /**
52	+ * Only use this in tests.
53	+ *
54	+ * @param Container $container
55	+ */
56	+ public static function push(Container $container)
57	+ {
58	+ self::$containerStack[] = $container;
59	+ }
60	+
61	+ public static function pop()
62	+ {
63	+ array_pop(self::$containerStack);
64	+ }
65	+
66	+ public static function addDefinitions(array $definitions)
67	+ {
68	+ self::$definitions[] = $definitions;
69	+ }
70	+
71	+ /**
72	+ * Proxy to Container::get()
73	+ *
74	+ * @param string $name Container entry name.
75	+ * @return mixed
76	+ * @throws \DI\NotFoundException
77	+ */
78	+ public static function get($name)
79	+ {
80	+ return self::getContainer()->get($name);
81	+ }
82	+
83	+ public static function getDefinitions()
84	+ {
85	+ return self::$definitions;
86	+ }
87	+ }

app/core/Context.php ADDED Viewed

	@@ -0,0 +1,94 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ */
8	+
9	+ namespace Piwik;
10	+
11	+ /**
12	+ * Methods related to changing the context Matomo code is running in.
13	+ */
14	+ class Context
15	+ {
16	+ public static function executeWithQueryParameters(array $parametersRequest, callable $callback)
17	+ {
18	+ // Temporarily sets the Request array to this API call context
19	+ $saveGET = $_GET;
20	+ $savePOST = $_POST;
21	+ $saveQUERY_STRING = @$_SERVER['QUERY_STRING'];
22	+ foreach ($parametersRequest as $param => $value) {
23	+ $_GET[$param] = $value;
24	+ $_POST[$param] = $value;
25	+ }
26	+
27	+ try {
28	+ return $callback();
29	+ } finally {
30	+ $_GET = $saveGET;
31	+ $_POST = $savePOST;
32	+ $_SERVER['QUERY_STRING'] = $saveQUERY_STRING;
33	+ }
34	+ }
35	+
36	+ /**
37	+ * Temporarily overwrites the idSite parameter so all code executed by `$callback()`
38	+ * will use that idSite.
39	+ *
40	+ * Useful when you need to change the idSite context for a chunk of code. For example,
41	+ * if we are archiving for more than one site in sequence, we don't want to use
42	+ * the same caches for both archiving executions.
43	+ *
44	+ * @param string\|int $idSite
45	+ * @param callable $callback
46	+ * @return mixed returns result of $callback
47	+ */
48	+ public static function changeIdSite($idSite, $callback)
49	+ {
50	+ // temporarily set the idSite query parameter so archiving will end up using
51	+ // the correct site aware caches
52	+ $originalGetIdSite = isset($_GET['idSite']) ? $_GET['idSite'] : null;
53	+ $originalPostIdSite = isset($_POST['idSite']) ? $_POST['idSite'] : null;
54	+
55	+ $originalGetIdSites = isset($_GET['idSites']) ? $_GET['idSites'] : null;
56	+ $originalPostIdSites = isset($_POST['idSites']) ? $_POST['idSites'] : null;
57	+
58	+ $originalTrackerGetIdSite = isset($_GET['idsite']) ? $_GET['idsite'] : null;
59	+ $originalTrackerPostIdSite = isset($_POST['idsite']) ? $_POST['idsite'] : null;
60	+
61	+ try {
62	+ $_GET['idSite'] = $_POST['idSite'] = $idSite;
63	+
64	+ if (Tracker::$initTrackerMode) {
65	+ $_GET['idsite'] = $_POST['idsite'] = $idSite;
66	+ }
67	+
68	+ // idSites is a deprecated query param that is still in use. since it is deprecated and new
69	+ // supported code shouldn't rely on it, we can (more) safely unset it here, since we are just
70	+ // calling downstream matomo code. we unset it because we don't want it interfering w/
71	+ // code in $callback().
72	+ unset($_GET['idSites']);
73	+ unset($_POST['idSites']);
74	+
75	+ return $callback();
76	+ } finally {
77	+ self::resetIdSiteParam($_GET, 'idSite', $originalGetIdSite);
78	+ self::resetIdSiteParam($_POST, 'idSite', $originalPostIdSite);
79	+ self::resetIdSiteParam($_GET, 'idSites', $originalGetIdSites);
80	+ self::resetIdSiteParam($_POST, 'idSites', $originalPostIdSites);
81	+ self::resetIdSiteParam($_GET, 'idsite', $originalTrackerGetIdSite);
82	+ self::resetIdSiteParam($_POST, 'idsite', $originalTrackerPostIdSite);
83	+ }
84	+ }
85	+
86	+ private static function resetIdSiteParam(&$superGlobal, $paramName, $originalValue)
87	+ {
88	+ if ($originalValue !== null) {
89	+ $superGlobal[$paramName] = $originalValue;
90	+ } else {
91	+ unset($superGlobal[$paramName]);
92	+ }
93	+ }
94	+ }

app/core/Cookie.php ADDED Viewed

	@@ -0,0 +1,462 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik;
10	+
11	+ use Piwik\Container\StaticContainer;
12	+
13	+ /**
14	+ * Simple class to handle the cookies:
15	+ * - read a cookie values
16	+ * - edit an existing cookie and save it
17	+ * - create a new cookie, set values, expiration date, etc. and save it
18	+ *
19	+ */
20	+ class Cookie
21	+ {
22	+ /**
23	+ * Don't create a cookie bigger than 1k
24	+ */
25	+ const MAX_COOKIE_SIZE = 1024;
26	+
27	+ /**
28	+ * The name of the cookie
29	+ * @var string
30	+ */
31	+ protected $name = null;
32	+
33	+ /**
34	+ * The expire time for the cookie (expressed in UNIX Timestamp)
35	+ * @var int
36	+ */
37	+ protected $expire = null;
38	+
39	+ /**
40	+ * Restrict cookie path
41	+ * @var string
42	+ */
43	+ protected $path = '';
44	+
45	+ /**
46	+ * Restrict cookie to a domain (or subdomains)
47	+ * @var string
48	+ */
49	+ protected $domain = '';
50	+
51	+ /**
52	+ * If true, cookie should only be transmitted over secure HTTPS
53	+ * @var bool
54	+ */
55	+ protected $secure = false;
56	+
57	+ /**
58	+ * If true, cookie will only be made available via the HTTP protocol.
59	+ * Note: not well supported by browsers.
60	+ * @var bool
61	+ */
62	+ protected $httponly = false;
63	+
64	+ /**
65	+ * The content of the cookie
66	+ * @var array
67	+ */
68	+ protected $value = array();
69	+
70	+ /**
71	+ * The character used to separate the tuple name=value in the cookie
72	+ */
73	+ const VALUE_SEPARATOR = ':';
74	+
75	+ /**
76	+ * Instantiate a new Cookie object and tries to load the cookie content if the cookie
77	+ * exists already.
78	+ *
79	+ * @param string $cookieName cookie Name
80	+ * @param int $expire The timestamp after which the cookie will expire, eg time() + 86400;
81	+ * use 0 (int zero) to expire cookie at end of browser session
82	+ * @param string $path The path on the server in which the cookie will be available on.
83	+ * @param bool\|string $keyStore Will be used to store several bits of data (eg. one array per website)
84	+ */
85	+ public function __construct($cookieName, $expire = null, $path = null, $keyStore = false)
86	+ {
87	+ $this->name = $cookieName;
88	+ $this->path = $path;
89	+ $this->expire = $expire;
90	+ if (is_null($expire)
91	+ \|\| !is_numeric($expire)
92	+ \|\| $expire < 0
93	+ ) {
94	+ $this->expire = $this->getDefaultExpire();
95	+ }
96	+
97	+ $this->keyStore = $keyStore;
98	+ if ($this->isCookieFound()) {
99	+ $this->loadContentFromCookie();
100	+ }
101	+ }
102	+
103	+ /**
104	+ * Returns true if the visitor already has the cookie.
105	+ *
106	+ * @return bool
107	+ */
108	+ public function isCookieFound()
109	+ {
110	+ return self::isCookieInRequest($this->name);
111	+ }
112	+
113	+ /**
114	+ * Returns the default expiry time, 2 years
115	+ *
116	+ * @return int Timestamp in 2 years
117	+ */
118	+ protected function getDefaultExpire()
119	+ {
120	+ return time() + 86400 * 365 * 2;
121	+ }
122	+
123	+ /**
124	+ * setcookie() replacement -- we don't use the built-in function because
125	+ * it is buggy for some PHP versions.
126	+ *
127	+ * @link http://php.net/setcookie
128	+ *
129	+ * @param string $Name Name of cookie
130	+ * @param string $Value Value of cookie
131	+ * @param int $Expires Time the cookie expires
132	+ * @param string $Path
133	+ * @param string $Domain
134	+ * @param bool $Secure
135	+ * @param bool $HTTPOnly
136	+ * @param string $sameSite
137	+ */
138	+ protected function setCookie($Name, $Value, $Expires, $Path = '', $Domain = '', $Secure = false, $HTTPOnly = false, $sameSite = false)
139	+ {
140	+ if (!empty($Domain)) {
141	+ // Fix the domain to accept domains with and without 'www.'.
142	+ if (!strncasecmp($Domain, 'www.', 4)) {
143	+ $Domain = substr($Domain, 4);
144	+ }
145	+ $Domain = '.' . $Domain;
146	+
147	+ // Remove port information.
148	+ $Port = strpos($Domain, ':');
149	+ if ($Port !== false) {
150	+ $Domain = substr($Domain, 0, $Port);
151	+ }
152	+ }
153	+
154	+ $header = 'Set-Cookie: ' . rawurlencode($Name) . '=' . rawurlencode($Value)
155	+ . (empty($Expires) ? '' : '; expires=' . gmdate('D, d-M-Y H:i:s', $Expires) . ' GMT')
156	+ . (empty($Path) ? '' : '; path=' . rawurlencode($Path))
157	+ . (empty($Domain) ? '' : '; domain=' . rawurlencode($Domain))
158	+ . (!$Secure ? '' : '; secure')
159	+ . (!$HTTPOnly ? '' : '; HttpOnly')
160	+ . (!$sameSite ? '' : '; SameSite=' . rawurlencode($sameSite));
161	+
162	+ Common::sendHeader($header, false);
163	+ }
164	+
165	+ /**
166	+ * We set the privacy policy header
167	+ */
168	+ protected function setP3PHeader()
169	+ {
170	+ Common::sendHeader("P3P: CP='OTI DSP COR NID STP UNI OTPa OUR'");
171	+ }
172	+
173	+ /**
174	+ * Delete the cookie
175	+ */
176	+ public function delete()
177	+ {
178	+ $this->setP3PHeader();
179	+ $this->setCookie($this->name, 'deleted', time() - 31536001, $this->path, $this->domain);
180	+ }
181	+
182	+ /**
183	+ * Saves the cookie (set the Cookie header).
184	+ * You have to call this method before sending any text to the browser or you would get the
185	+ * "Header already sent" error.
186	+ * @param string $sameSite Value for SameSite cookie property
187	+ */
188	+ public function save($sameSite = null)
189	+ {
190	+ if ($sameSite) {
191	+ $sameSite = self::getSameSiteValueForBrowser($sameSite);
192	+ }
193	+ $cookieString = $this->generateContentString();
194	+ if (strlen($cookieString) > self::MAX_COOKIE_SIZE) {
195	+ // If the cookie was going to be too large, instead, delete existing cookie and start afresh
196	+ $this->delete();
197	+ return;
198	+ }
199	+
200	+ $this->setP3PHeader();
201	+ $this->setCookie($this->name, $cookieString, $this->expire, $this->path, $this->domain, $this->secure, $this->httponly, $sameSite);
202	+ }
203	+
204	+ /**
205	+ * Extract signed content from string: content VALUE_SEPARATOR '_=' signature
206	+ *
207	+ * @param string $content
208	+ * @return string\|bool Content or false if unsigned
209	+ */
210	+ private function extractSignedContent($content)
211	+ {
212	+ $signature = substr($content, -40);
213	+
214	+ if (substr($content, -43, 3) == self::VALUE_SEPARATOR . '_=' &&
215	+ $signature === sha1(substr($content, 0, -40) . SettingsPiwik::getSalt())
216	+ ) {
217	+ // strip trailing: VALUE_SEPARATOR '_=' signature"
218	+ return substr($content, 0, -43);
219	+ }
220	+
221	+ return false;
222	+ }
223	+
224	+ /**
225	+ * Load the cookie content into a php array.
226	+ * Parses the cookie string to extract the different variables.
227	+ * Unserialize the array when necessary.
228	+ * Decode the non numeric values that were base64 encoded.
229	+ */
230	+ protected function loadContentFromCookie()
231	+ {
232	+ $cookieStr = $this->extractSignedContent($_COOKIE[$this->name]);
233	+
234	+ if ($cookieStr === false) {
235	+ return;
236	+ }
237	+
238	+ $values = explode(self::VALUE_SEPARATOR, $cookieStr);
239	+ foreach ($values as $nameValue) {
240	+ $equalPos = strpos($nameValue, '=');
241	+ $varName = substr($nameValue, 0, $equalPos);
242	+ $varValue = substr($nameValue, $equalPos + 1);
243	+
244	+ // no numeric value are base64 encoded so we need to decode them
245	+ if (!is_numeric($varValue)) {
246	+ $tmpValue = base64_decode($varValue);
247	+ $varValue = safe_unserialize($tmpValue);
248	+
249	+ // discard entire cookie
250	+ // note: this assumes we never serialize a boolean
251	+ if ($varValue === false && $tmpValue !== 'b:0;') {
252	+ $this->value = array();
253	+ unset($_COOKIE[$this->name]);
254	+ break;
255	+ }
256	+ }
257	+
258	+ $this->value[$varName] = $varValue;
259	+ }
260	+ }
261	+
262	+ /**
263	+ * Returns the string to save in the cookie from the $this->value array of values.
264	+ * It goes through the array and generates the cookie content string.
265	+ *
266	+ * @return string Cookie content
267	+ */
268	+ public function generateContentString()
269	+ {
270	+ $cookieStr = '';
271	+
272	+ foreach ($this->value as $name => $value) {
273	+ if (!is_numeric($value)) {
274	+ $value = base64_encode(safe_serialize($value));
275	+ }
276	+
277	+ $cookieStr .= "$name=$value" . self::VALUE_SEPARATOR;
278	+ }
279	+
280	+ if (!empty($cookieStr)) {
281	+ $cookieStr .= '_=';
282	+
283	+ // sign cookie
284	+ $signature = sha1($cookieStr . SettingsPiwik::getSalt());
285	+ return $cookieStr . $signature;
286	+ }
287	+
288	+ return '';
289	+ }
290	+
291	+ /**
292	+ * Set cookie domain
293	+ *
294	+ * @param string $domain
295	+ */
296	+ public function setDomain($domain)
297	+ {
298	+ $this->domain = $domain;
299	+ }
300	+
301	+ /**
302	+ * Set secure flag
303	+ *
304	+ * @param bool $secure
305	+ */
306	+ public function setSecure($secure)
307	+ {
308	+ $this->secure = $secure;
309	+ }
310	+
311	+ /**
312	+ * Set HTTP only
313	+ *
314	+ * @param bool $httponly
315	+ */
316	+ public function setHttpOnly($httponly)
317	+ {
318	+ $this->httponly = $httponly;
319	+ }
320	+
321	+ /**
322	+ * Registers a new name => value association in the cookie.
323	+ *
324	+ * Registering new values is optimal if the value is a numeric value.
325	+ * If the value is a string, it will be saved as a base64 encoded string.
326	+ * If the value is an array, it will be saved as a serialized and base64 encoded
327	+ * string which is not very good in terms of bytes usage.
328	+ * You should save arrays only when you are sure about their maximum data size.
329	+ * A cookie has to stay small and its size shouldn't increase over time!
330	+ *
331	+ * @param string $name Name of the value to save; the name will be used to retrieve this value
332	+ * @param string\|array\|number $value Value to save. If null, entry will be deleted from cookie.
333	+ */
334	+ public function set($name, $value)
335	+ {
336	+ $name = self::escapeValue($name);
337	+
338	+ // Delete value if $value === null
339	+ if (is_null($value)) {
340	+ if ($this->keyStore === false) {
341	+ unset($this->value[$name]);
342	+ return;
343	+ }
344	+ unset($this->value[$this->keyStore][$name]);
345	+ return;
346	+ }
347	+
348	+ if ($this->keyStore === false) {
349	+ $this->value[$name] = $value;
350	+ return;
351	+ }
352	+
353	+ $this->value[$this->keyStore][$name] = $value;
354	+ }
355	+
356	+ /**
357	+ * Returns the value defined by $name from the cookie.
358	+ *
359	+ * @param string\|integer Index name of the value to return
360	+ * @return mixed The value if found, false if the value is not found
361	+ */
362	+ public function get($name)
363	+ {
364	+ $name = self::escapeValue($name);
365	+ if (false === $this->keyStore) {
366	+ if (isset($this->value[$name])) {
367	+ return self::escapeValue($this->value[$name]);
368	+ }
369	+
370	+ return false;
371	+ }
372	+
373	+ if (isset($this->value[$this->keyStore][$name])) {
374	+ return self::escapeValue($this->value[$this->keyStore][$name]);
375	+ }
376	+
377	+ return false;
378	+ }
379	+
380	+ /**
381	+ * Removes all values from the cookie.
382	+ */
383	+ public function clear()
384	+ {
385	+ $this->value = [];
386	+ }
387	+
388	+ /**
389	+ * Returns an easy to read cookie dump
390	+ *
391	+ * @return string The cookie dump
392	+ */
393	+ public function __toString()
394	+ {
395	+ $str = 'COOKIE ' . $this->name . ', rows count: ' . count($this->value) . ', cookie size = ' . strlen($this->generateContentString()) . " bytes, ";
396	+ $str .= 'path: ' . $this->path. ', expire: ' . $this->expire . "\n";
397	+ $str .= var_export($this->value, $return = true);
398	+
399	+ return $str;
400	+ }
401	+
402	+ /**
403	+ * Escape values from the cookie before sending them back to the client
404	+ * (when using the get() method).
405	+ *
406	+ * @param string $value Value to be escaped
407	+ * @return mixed The value once cleaned.
408	+ */
409	+ protected static function escapeValue($value)
410	+ {
411	+ return Common::sanitizeInputValues($value);
412	+ }
413	+
414	+ /**
415	+ * Returns true if a cookie named '$name' is in the current HTTP request,
416	+ * false if otherwise.
417	+ *
418	+ * @param string $name the name of the cookie
419	+ * @return boolean
420	+ */
421	+ public static function isCookieInRequest($name)
422	+ {
423	+ return isset($_COOKIE[$name]);
424	+ }
425	+
426	+ /**
427	+ * Find the most suitable value for a cookie SameSite attribute, given environmental restrictions which
428	+ * may make the most "correct" value impractical:
429	+ * - On Chrome, the "None" value means that the cookie will not be present on third-party sites (e.g. the site
430	+ * that is being tracked) when the site is loaded over HTTP. This means that important cookies which should always
431	+ * be present (e.g. the opt-out cookie) won't be there at all. Using "Lax" means that at least they will be there
432	+ * for some requests which are deemed CSRF-safe, although other requests may have broken functionality.
433	+ * - On Safari, the "None" value is interpreted as "Strict". In order to set a cookie which will be available
434	+ * in all third-party contexts, we have to omit the SameSite attribute altogether.
435	+ * @param string $default The desired SameSite value that we should use if it won't cause any problems.
436	+ * @return string SameSite attribute value that should be set on the cookie. Empty string indicates that no value
437	+ * should be set.
438	+ */
439	+ private static function getSameSiteValueForBrowser($default)
440	+ {
441	+ $sameSite = ucfirst(strtolower($default));
442	+
443	+ if ($sameSite == 'None') {
444	+ $userAgent = Http::getUserAgent();
445	+ $ddFactory = StaticContainer::get(\Piwik\DeviceDetector\DeviceDetectorFactory::class);
446	+ $deviceDetector = $ddFactory->makeInstance($userAgent);
447	+ $deviceDetector->parse();
448	+ $browser = $deviceDetector->getClient();
449	+ if (is_array($browser)) {
450	+ $browser = $browser['name'];
451	+ }
452	+
453	+ if ((!ProxyHttp::isHttps()) && $browser === 'Chrome') {
454	+ $sameSite = 'Lax';
455	+ } else if ($browser === 'Safari') {
456	+ $sameSite = '';
457	+ }
458	+ }
459	+
460	+ return $sameSite;
461	+ }
462	+ }

app/core/CronArchive.php ADDED Viewed

	@@ -0,0 +1,2192 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik;
10	+
11	+ use Exception;
12	+ use Piwik\ArchiveProcessor\Parameters;
13	+ use Piwik\ArchiveProcessor\PluginsArchiver;
14	+ use Piwik\ArchiveProcessor\Rules;
15	+ use Piwik\Archiver\Request;
16	+ use Piwik\CliMulti\Process;
17	+ use Piwik\Container\StaticContainer;
18	+ use Piwik\CronArchive\FixedSiteIds;
19	+ use Piwik\CronArchive\Performance\Logger;
20	+ use Piwik\CronArchive\SharedSiteIds;
21	+ use Piwik\Archive\ArchiveInvalidator;
22	+ use Piwik\DataAccess\ArchiveSelector;
23	+ use Piwik\DataAccess\RawLogDao;
24	+ use Piwik\Exception\UnexpectedWebsiteFoundException;
25	+ use Piwik\Metrics\Formatter;
26	+ use Piwik\Period\Factory;
27	+ use Piwik\Period\Factory as PeriodFactory;
28	+ use Piwik\CronArchive\SitesToReprocessDistributedList;
29	+ use Piwik\CronArchive\SegmentArchivingRequestUrlProvider;
30	+ use Piwik\Period\Range;
31	+ use Piwik\Plugins\CoreAdminHome\API as CoreAdminHomeAPI;
32	+ use Piwik\Plugins\SegmentEditor\Model as SegmentEditorModel;
33	+ use Piwik\Plugins\SitesManager\API as APISitesManager;
34	+ use Piwik\Plugins\UsersManager\API as APIUsersManager;
35	+ use Piwik\Plugins\UsersManager\UserPreferences;
36	+ use Psr\Log\LoggerInterface;
37	+
38	+ /**
39	+ * ./console core:archive runs as a cron and is a useful tool for general maintenance,
40	+ * and pre-process reports for a Fast dashboard rendering.
41	+ */
42	+ class CronArchive
43	+ {
44	+ // the url can be set here before the init, and it will be used instead of --url=
45	+ public static $url = false;
46	+
47	+ // Max parallel requests for a same site's segments
48	+ const MAX_CONCURRENT_API_REQUESTS = 3;
49	+
50	+ // force-timeout-for-periods default (1 hour)
51	+ const SECONDS_DELAY_BETWEEN_PERIOD_ARCHIVES = 3600;
52	+
53	+ // force-all-periods default (7 days)
54	+ const ARCHIVE_SITES_WITH_TRAFFIC_SINCE = 604800;
55	+
56	+ // By default, will process last 52 days and months
57	+ // It will be overwritten by the number of days since last archiving ran until completion.
58	+ const DEFAULT_DATE_LAST = 52;
59	+
60	+ // Since weeks are not used in yearly archives, we make sure that all possible weeks are processed
61	+ const DEFAULT_DATE_LAST_WEEKS = 260;
62	+
63	+ const DEFAULT_DATE_LAST_YEARS = 7;
64	+
65	+ // Flag to know when the archive cron is calling the API
66	+ const APPEND_TO_API_REQUEST = '&trigger=archivephp';
67	+
68	+ // Flag used to record timestamp in Option::
69	+ const OPTION_ARCHIVING_FINISHED_TS = "LastCompletedFullArchiving";
70	+
71	+ // Name of option used to store starting timestamp
72	+ const OPTION_ARCHIVING_STARTED_TS = "LastFullArchivingStartTime";
73	+
74	+ // Show only first N characters from Piwik API output in case of errors
75	+ const TRUNCATE_ERROR_MESSAGE_SUMMARY = 6000;
76	+
77	+ // archiving will be triggered on all websites with traffic in the last $shouldArchiveOnlySitesWithTrafficSince seconds
78	+ private $shouldArchiveOnlySitesWithTrafficSince;
79	+
80	+ // By default, we only process the current week/month/year at most once an hour
81	+ private $processPeriodsMaximumEverySeconds;
82	+ private $todayArchiveTimeToLive;
83	+ private $websiteDayHasFinishedSinceLastRun = array();
84	+ private $idSitesInvalidatedOldReports = array();
85	+ private $shouldArchiveOnlySpecificPeriods = array();
86	+ private $idSitesNotUsingTracker;
87	+
88	+ /**
89	+ * @var SharedSiteIds\|FixedSiteIds
90	+ */
91	+ private $websites = array();
92	+ private $allWebsites = array();
93	+ private $segments = array();
94	+ private $visitsToday = 0;
95	+ private $requests = 0;
96	+ private $archiveAndRespectTTL = true;
97	+
98	+ private $lastSuccessRunTimestamp = false;
99	+ private $errors = array();
100	+
101	+ private $apiToInvalidateArchivedReport;
102	+
103	+ const NO_ERROR = "no error";
104	+
105	+ public $testmode = false;
106	+
107	+ /**
108	+ * The list of IDs for sites for whom archiving should be initiated. If supplied, only these
109	+ * sites will be archived.
110	+ *
111	+ * @var int[]
112	+ */
113	+ public $shouldArchiveSpecifiedSites = array();
114	+
115	+ /**
116	+ * The list of IDs of sites to ignore when launching archiving. Archiving will not be launched
117	+ * for any site whose ID is in this list (even if the ID is supplied in {@link $shouldArchiveSpecifiedSites}
118	+ * or if {@link $shouldArchiveAllSites} is true).
119	+ *
120	+ * @var int[]
121	+ */
122	+ public $shouldSkipSpecifiedSites = array();
123	+
124	+ /**
125	+ * If true, archiving will be launched for every site.
126	+ *
127	+ * @var bool
128	+ */
129	+ public $shouldArchiveAllSites = false;
130	+
131	+ /**
132	+ * If true, xhprof will be initiated for the archiving run. Only for development/testing.
133	+ *
134	+ * @var bool
135	+ */
136	+ public $shouldStartProfiler = false;
137	+
138	+ /**
139	+ * Given options will be forwarded to the PHP command if the archiver is executed via CLI.
140	+ * @var string
141	+ */
142	+ public $phpCliConfigurationOptions = '';
143	+
144	+ /**
145	+ * If HTTP requests are used to initiate archiving, this controls whether invalid SSL certificates should
146	+ * be accepted or not by each request.
147	+ *
148	+ * @var bool
149	+ */
150	+ public $acceptInvalidSSLCertificate = false;
151	+
152	+ /**
153	+ * If set to true, scheduled tasks will not be run.
154	+ *
155	+ * @var bool
156	+ */
157	+ public $disableScheduledTasks = false;
158	+
159	+ /**
160	+ * The amount of seconds between non-day period archiving. That is, if archiving has been launched within
161	+ * the past [$forceTimeoutPeriod] seconds, Piwik will not initiate archiving for week, month and year periods.
162	+ *
163	+ * @var int\|false
164	+ */
165	+ public $forceTimeoutPeriod = false;
166	+
167	+ /**
168	+ * If supplied, archiving will be launched for sites that have had visits within the last [$shouldArchiveAllPeriodsSince]
169	+ * seconds. If set to `true`, the value defaults to {@link ARCHIVE_SITES_WITH_TRAFFIC_SINCE}.
170	+ *
171	+ * @var int\|bool
172	+ */
173	+ public $shouldArchiveAllPeriodsSince = false;
174	+
175	+ /**
176	+ * If supplied, archiving will be launched only for periods that fall within this date range. For example,
177	+ * `"2012-01-01,2012-03-15"` would result in January 2012, February 2012 being archived but not April 2012.
178	+ *
179	+ * @var string\|false eg, `"2012-01-01,2012-03-15"`
180	+ */
181	+ public $restrictToDateRange = false;
182	+
183	+ /**
184	+ * A list of periods to launch archiving for. By default, day, week, month and year periods
185	+ * are considered. This variable can limit the periods to, for example, week & month only.
186	+ *
187	+ * @var string[] eg, `array("day","week","month","year")`
188	+ */
189	+ public $restrictToPeriods = array();
190	+
191	+ /**
192	+ * Forces CronArchive to retrieve data for the last [$dateLastForced] periods when initiating archiving.
193	+ * When archiving weeks, for example, if 10 is supplied, the API will be called w/ last10. This will potentially
194	+ * initiate archiving for the last 10 weeks.
195	+ *
196	+ * @var int\|false
197	+ */
198	+ public $dateLastForced = false;
199	+
200	+ /**
201	+ * The number of concurrent requests to issue per website. Defaults to {@link MAX_CONCURRENT_API_REQUESTS}.
202	+ *
203	+ * Used when archiving a site's segments concurrently.
204	+ *
205	+ * @var int\|false
206	+ */
207	+ public $concurrentRequestsPerWebsite = false;
208	+
209	+ /**
210	+ * The number of concurrent archivers to run at once max.
211	+ *
212	+ * @var int\|false
213	+ */
214	+ public $maxConcurrentArchivers = false;
215	+
216	+ /**
217	+ * List of segment strings to force archiving for. If a stored segment is not in this list, it will not
218	+ * be archived.
219	+ *
220	+ * @var string[]
221	+ */
222	+ public $segmentsToForce = array();
223	+
224	+ /**
225	+ * @var bool
226	+ */
227	+ public $disableSegmentsArchiving = false;
228	+
229	+ /**
230	+ * If enabled, segments will be only archived for yesterday, but not today. If the segment was created recently,
231	+ * then it will still be archived for today and the setting will be ignored for this segment.
232	+ * @var bool
233	+ */
234	+ public $skipSegmentsToday = false;
235	+
236	+ private $websitesWithVisitsSinceLastRun = 0;
237	+ private $skippedPeriodsArchivesWebsite = 0;
238	+ private $skippedPeriodsNoDataInPeriod = 0;
239	+ private $skippedDayArchivesWebsites = 0;
240	+ private $skippedDayNoRecentData = 0;
241	+ private $skippedDayOnApiError = 0;
242	+ private $skipped = 0;
243	+ private $processed = 0;
244	+ private $archivedPeriodsArchivesWebsite = 0;
245	+
246	+ private $archivingStartingTime;
247	+
248	+ private $formatter;
249	+
250	+ /**
251	+ * @var SegmentArchivingRequestUrlProvider
252	+ */
253	+ private $segmentArchivingRequestUrlProvider;
254	+
255	+ /**
256	+ * @var LoggerInterface
257	+ */
258	+ private $logger;
259	+
260	+ /**
261	+ * Only used when archiving using HTTP requests.
262	+ *
263	+ * @var string
264	+ */
265	+ private $urlToPiwik = null;
266	+
267	+ /**
268	+ * @var ArchiveInvalidator
269	+ */
270	+ private $invalidator;
271	+
272	+ /**
273	+ * @var bool
274	+ */
275	+ private $isArchiveProfilingEnabled = false;
276	+
277	+ /**
278	+ * Returns the option name of the option that stores the time core:archive was last executed.
279	+ *
280	+ * @param int $idSite
281	+ * @param string $period
282	+ * @return string
283	+ */
284	+ public static function lastRunKey($idSite, $period)
285	+ {
286	+ return "lastRunArchive" . $period . "_" . $idSite;
287	+ }
288	+
289	+ /**
290	+ * Constructor.
291	+ *
292	+ * @param string\|null $processNewSegmentsFrom When to archive new segments from. See [General] process_new_segments_from
293	+ * for possible values.
294	+ * @param LoggerInterface\|null $logger
295	+ */
296	+ public function __construct($processNewSegmentsFrom = null, LoggerInterface $logger = null)
297	+ {
298	+ $this->logger = $logger ?: StaticContainer::get('Psr\Log\LoggerInterface');
299	+ $this->formatter = new Formatter();
300	+
301	+ $processNewSegmentsFrom = $processNewSegmentsFrom ?: StaticContainer::get('ini.General.process_new_segments_from');
302	+
303	+ $this->segmentArchivingRequestUrlProvider = new SegmentArchivingRequestUrlProvider($processNewSegmentsFrom);
304	+
305	+ $this->invalidator = StaticContainer::get('Piwik\Archive\ArchiveInvalidator');
306	+
307	+ $this->isArchiveProfilingEnabled = Config::getInstance()->Debug['archiving_profile'] == 1;
308	+ }
309	+
310	+ private function isMaintenanceModeEnabled()
311	+ {
312	+ return Config::getInstance()->General['maintenance_mode'] == 1;
313	+ }
314	+
315	+ /**
316	+ * Initializes and runs the cron archiver.
317	+ */
318	+ public function main()
319	+ {
320	+ if ($this->isMaintenanceModeEnabled()) {
321	+ $this->logger->info("Archiving won't run because maintenance mode is enabled");
322	+ return;
323	+ }
324	+
325	+ $self = $this;
326	+ Access::doAsSuperUser(function () use ($self) {
327	+ $self->init();
328	+ $self->run();
329	+ $self->runScheduledTasks();
330	+ $self->end();
331	+ });
332	+ }
333	+
334	+ public function init()
335	+ {
336	+ /**
337	+ * This event is triggered during initializing archiving.
338	+ *
339	+ * @param CronArchive $this
340	+ */
341	+ Piwik::postEvent('CronArchive.init.start', array($this));
342	+
343	+ SettingsServer::setMaxExecutionTime(0);
344	+
345	+ $this->archivingStartingTime = time();
346	+
347	+ // Note: the order of methods call matters here.
348	+ $this->initStateFromParameters();
349	+
350	+ $this->logInitInfo();
351	+ $this->logArchiveTimeoutInfo();
352	+
353	+ // record archiving start time
354	+ Option::set(self::OPTION_ARCHIVING_STARTED_TS, time());
355	+
356	+ $this->segments = $this->initSegmentsToArchive();
357	+ $this->allWebsites = APISitesManager::getInstance()->getAllSitesId();
358	+
359	+ if (!empty($this->shouldArchiveOnlySpecificPeriods)) {
360	+ $this->logger->info("- Will only process the following periods: " . implode(", ", $this->shouldArchiveOnlySpecificPeriods) . " (--force-periods)");
361	+ }
362	+
363	+ $this->invalidateArchivedReportsForSitesThatNeedToBeArchivedAgain();
364	+
365	+ $websitesIds = $this->initWebsiteIds();
366	+ $this->filterWebsiteIds($websitesIds);
367	+
368	+ $this->websites = $this->createSitesToArchiveQueue($websitesIds);
369	+
370	+ if ($this->websites->getInitialSiteIds() != $websitesIds) {
371	+ $this->logger->info('Will ignore websites and help finish a previous started queue instead. IDs: ' . implode(', ', $this->websites->getInitialSiteIds()));
372	+ }
373	+
374	+ if ($this->skipSegmentsToday) {
375	+ $this->logger->info('Will skip segments archiving for today unless they were created recently');
376	+ }
377	+
378	+ $this->logForcedSegmentInfo();
379	+
380	+ /**
381	+ * This event is triggered after a CronArchive instance is initialized.
382	+ *
383	+ * @param array $websiteIds The list of website IDs this CronArchive instance is processing.
384	+ * This will be the entire list of IDs regardless of whether some have
385	+ * already been processed.
386	+ */
387	+ Piwik::postEvent('CronArchive.init.finish', array($this->websites->getInitialSiteIds()));
388	+ }
389	+
390	+ /**
391	+ * Main function, runs archiving on all websites with new activity
392	+ */
393	+ public function run()
394	+ {
395	+ $timer = new Timer;
396	+
397	+ $this->logSection("START");
398	+ $this->logger->info("Starting Matomo reports archiving...");
399	+
400	+ $numWebsitesScheduled = $this->websites->getNumSites();
401	+ $numWebsitesArchived = 0;
402	+
403	+ $cliMulti = $this->makeCliMulti();
404	+ if ($this->maxConcurrentArchivers && $cliMulti->supportsAsync()) {
405	+ $numRunning = 0;
406	+ $processes = Process::getListOfRunningProcesses();
407	+ $instanceId = SettingsPiwik::getPiwikInstanceId();
408	+
409	+ foreach ($processes as $process) {
410	+ if (strpos($process, 'console core:archive') !== false &&
411	+ (!$instanceId
412	+ \|\| strpos($process, '--matomo-domain=' . $instanceId) !== false
413	+ \|\| strpos($process, '--matomo-domain="' . $instanceId . '"') !== false
414	+ \|\| strpos($process, '--matomo-domain=\'' . $instanceId . "'") !== false
415	+ \|\| strpos($process, '--piwik-domain=' . $instanceId) !== false
416	+ \|\| strpos($process, '--piwik-domain="' . $instanceId . '"') !== false
417	+ \|\| strpos($process, '--piwik-domain=\'' . $instanceId . "'") !== false)) {
418	+ $numRunning++;
419	+ }
420	+ }
421	+ if ($this->maxConcurrentArchivers < $numRunning) {
422	+ $this->logger->info(sprintf("Archiving will stop now because %s archivers are already running and max %s are supposed to run at once.", $numRunning, $this->maxConcurrentArchivers));
423	+ return;
424	+ } else {
425	+ $this->logger->info(sprintf("%s out of %s archivers running currently", $numRunning, $this->maxConcurrentArchivers));
426	+ }
427	+ }
428	+
429	+ do {
430	+ if ($this->isMaintenanceModeEnabled()) {
431	+ $this->logger->info("Archiving will stop now because maintenance mode is enabled");
432	+ return;
433	+ }
434	+
435	+ $idSite = $this->websites->getNextSiteId();
436	+ $numWebsitesArchived++;
437	+
438	+ if (null === $idSite) {
439	+ break;
440	+ }
441	+
442	+ if ($numWebsitesArchived > $numWebsitesScheduled) {
443	+ // this is needed because a cron:archive might run for example for 5 hours. Meanwhile 5 other
444	+ // `cron:archive` have been possibly started... this means meanwhile, within the 5 hours, the
445	+ // `list of SharedSiteIds` have been potentially emptied and filled again from the beginning.
446	+ // This means 5 hours later, even though all websites that were originally in the list have been
447	+ // finished by now, the `cron:archive` will stay active and continue processing because the list of
448	+ // siteIds to archive was resetted by another `cron:archive` command. Potentially some `cron:archive`
449	+ // will basically never end because by the time the `cron:archive` finishes, the sharedSideIds have
450	+ // been resettet. This can eventually lead to some random concurrency issues when there are like
451	+ // 40 `core:archive` active at the same time.
452	+ $this->logger->info("Stopping archiving as the initial list of websites has been processed.");
453	+ return;
454	+ }
455	+
456	+ if (!Process::isMethodDisabled('getmypid') && !Process::isMethodDisabled('ignore_user_abort')) {
457	+ // see https://github.com/matomo-org/wp-matomo/issues/163
458	+ flush();
459	+ }
460	+
461	+ $requestsBefore = $this->requests;
462	+ if ($idSite <= 0) {
463	+ continue;
464	+ }
465	+
466	+ $skipWebsiteForced = in_array($idSite, $this->shouldSkipSpecifiedSites);
467	+ if ($skipWebsiteForced) {
468	+ $this->logger->info("Skipped website id $idSite, found in --skip-idsites ");
469	+ $this->skipped++;
470	+ continue;
471	+ }
472	+
473	+ $shouldCheckIfArchivingIsNeeded = !$this->shouldArchiveSpecifiedSites && !$this->shouldArchiveAllSites && !$this->dateLastForced;
474	+ $hasWebsiteDayFinishedSinceLastRun = in_array($idSite, $this->websiteDayHasFinishedSinceLastRun);
475	+ $isOldReportInvalidatedForWebsite = $this->isOldReportInvalidatedForWebsite($idSite);
476	+
477	+ if ($shouldCheckIfArchivingIsNeeded) {
478	+ // if not specific sites and not all websites should be archived, we check whether we actually have
479	+ // to process the archives for this website (only if there were visits since midnight)
480	+ if (!$hasWebsiteDayFinishedSinceLastRun && !$isOldReportInvalidatedForWebsite) {
481	+
482	+ try {
483	+ if ($this->isWebsiteUsingTheTracker($idSite)) {
484	+
485	+ if (!$this->hadWebsiteTrafficSinceMidnightInTimezone($idSite)) {
486	+ $this->logger->info("Skipped website id $idSite as archiving is not needed");
487	+
488	+ $this->skippedDayNoRecentData++;
489	+ $this->skipped++;
490	+ continue;
491	+ }
492	+ } else {
493	+ $this->logger->info("- website id $idSite is not using the tracker");
494	+ }
495	+ } catch (UnexpectedWebsiteFoundException $e) {
496	+ $this->logger->info("Skipped website id $idSite, got: UnexpectedWebsiteFoundException");
497	+ continue;
498	+ }
499	+
500	+ } elseif ($hasWebsiteDayFinishedSinceLastRun) {
501	+ $this->logger->info("Day has finished for website id $idSite since last run");
502	+ } elseif ($isOldReportInvalidatedForWebsite) {
503	+ $this->logger->info("Old report was invalidated for website id $idSite");
504	+ }
505	+ }
506	+
507	+ /**
508	+ * This event is triggered before the cron archiving process starts archiving data for a single
509	+ * site.
510	+ *
511	+ * @param int $idSite The ID of the site we're archiving data for.
512	+ */
513	+ Piwik::postEvent('CronArchive.archiveSingleSite.start', array($idSite));
514	+
515	+ $completed = $this->archiveSingleSite($idSite, $requestsBefore);
516	+
517	+ /**
518	+ * This event is triggered immediately after the cron archiving process starts archiving data for a single
519	+ * site.
520	+ *
521	+ * @param int $idSite The ID of the site we're archiving data for.
522	+ */
523	+ Piwik::postEvent('CronArchive.archiveSingleSite.finish', array($idSite, $completed));
524	+ } while (!empty($idSite));
525	+
526	+ $this->logger->info("Done archiving!");
527	+
528	+ $this->logSection("SUMMARY");
529	+ $this->logger->info("Total visits for today across archived websites: " . $this->visitsToday);
530	+
531	+ $totalWebsites = count($this->allWebsites);
532	+ $this->skipped = $totalWebsites - $this->websitesWithVisitsSinceLastRun;
533	+ $this->logger->info("Archived today's reports for {$this->websitesWithVisitsSinceLastRun} websites");
534	+ $this->logger->info("Archived week/month/year for {$this->archivedPeriodsArchivesWebsite} websites");
535	+ $this->logger->info("Skipped {$this->skipped} websites");
536	+ $this->logger->info("- {$this->skippedDayNoRecentData} skipped because no new visit since the last script execution");
537	+ $this->logger->info("- {$this->skippedDayArchivesWebsites} skipped because existing daily reports are less than {$this->todayArchiveTimeToLive} seconds old");
538	+ $this->logger->info("- {$this->skippedPeriodsArchivesWebsite} skipped because existing week/month/year periods reports are less than {$this->processPeriodsMaximumEverySeconds} seconds old");
539	+
540	+ if($this->skippedPeriodsNoDataInPeriod) {
541	+ $this->logger->info("- {$this->skippedPeriodsNoDataInPeriod} skipped periods archiving because no visit in recent days");
542	+ }
543	+
544	+ if($this->skippedDayOnApiError) {
545	+ $this->logger->info("- {$this->skippedDayOnApiError} skipped because got an error while querying reporting API");
546	+ }
547	+ $this->logger->info("Total API requests: {$this->requests}");
548	+
549	+ //DONE: done/total, visits, wtoday, wperiods, reqs, time, errors[count]: first eg.
550	+ $percent = $this->websites->getNumSites() == 0
551	+ ? ""
552	+ : " " . round($this->processed * 100 / $this->websites->getNumSites(), 0) . "%";
553	+ $this->logger->info("done: " .
554	+ $this->processed . "/" . $this->websites->getNumSites() . "" . $percent . ", " .
555	+ $this->visitsToday . " vtoday, $this->websitesWithVisitsSinceLastRun wtoday, {$this->archivedPeriodsArchivesWebsite} wperiods, " .
556	+ $this->requests . " req, " . round($timer->getTimeMs()) . " ms, " .
557	+ (empty($this->errors)
558	+ ? self::NO_ERROR
559	+ : (count($this->errors) . " errors."))
560	+ );
561	+
562	+ $this->logger->info($timer->__toString());
563	+ }
564	+
565	+ public function getErrors()
566	+ {
567	+ return $this->errors;
568	+ }
569	+
570	+ /**
571	+ * End of the script
572	+ */
573	+ public function end()
574	+ {
575	+ /**
576	+ * This event is triggered after archiving.
577	+ *
578	+ * @param CronArchive $this
579	+ */
580	+ Piwik::postEvent('CronArchive.end', array($this));
581	+
582	+ if (empty($this->errors)) {
583	+ // No error -> Logs the successful script execution until completion
584	+ Option::set(self::OPTION_ARCHIVING_FINISHED_TS, time());
585	+ return;
586	+ }
587	+
588	+ $this->logSection("SUMMARY OF ERRORS");
589	+ foreach ($this->errors as $error) {
590	+ // do not logError since errors are already in stderr
591	+ $this->logger->info("Error: " . $error);
592	+ }
593	+
594	+ $summary = count($this->errors) . " total errors during this script execution, please investigate and try and fix these errors.";
595	+ $this->logFatalError($summary);
596	+ }
597	+
598	+ public function logFatalError($m)
599	+ {
600	+ $this->logError($m);
601	+
602	+ throw new Exception($m);
603	+ }
604	+
605	+ /**
606	+ * @param int[] $idSegments
607	+ */
608	+ public function setSegmentsToForceFromSegmentIds($idSegments)
609	+ {
610	+ /** @var SegmentEditorModel $segmentEditorModel */
611	+ $segmentEditorModel = StaticContainer::get('Piwik\Plugins\SegmentEditor\Model');
612	+ $segments = $segmentEditorModel->getAllSegmentsAndIgnoreVisibility();
613	+
614	+ $segments = array_filter($segments, function ($segment) use ($idSegments) {
615	+ return in_array($segment['idsegment'], $idSegments);
616	+ });
617	+
618	+ $segments = array_map(function ($segment) {
619	+ return $segment['definition'];
620	+ }, $segments);
621	+
622	+ $this->segmentsToForce = $segments;
623	+ }
624	+
625	+ public function runScheduledTasks()
626	+ {
627	+ $this->logSection("SCHEDULED TASKS");
628	+
629	+ if ($this->disableScheduledTasks) {
630	+ $this->logger->info("Scheduled tasks are disabled with --disable-scheduled-tasks");
631	+ return;
632	+ }
633	+
634	+ // TODO: this is a HACK to get the purgeOutdatedArchives task to work when run below. without
635	+ // it, the task will not run because we no longer run the tasks through CliMulti.
636	+ // harder to implement alternatives include:
637	+ // - moving CronArchive logic to DI and setting a flag in the class when the whole process
638	+ // runs
639	+ // - setting a new DI environment for core:archive which CoreAdminHome can use to conditionally
640	+ // enable/disable the task
641	+ $_GET['trigger'] = 'archivephp';
642	+ CoreAdminHomeAPI::getInstance()->runScheduledTasks();
643	+
644	+ $this->logSection("");
645	+ }
646	+
647	+ private function archiveSingleSite($idSite, $requestsBefore)
648	+ {
649	+ $timerWebsite = new Timer;
650	+
651	+ $lastTimestampWebsiteProcessedPeriods = $lastTimestampWebsiteProcessedDay = false;
652	+
653	+ if ($this->archiveAndRespectTTL) {
654	+ Option::clearCachedOption($this->lastRunKey($idSite, "periods"));
655	+ $lastTimestampWebsiteProcessedPeriods = $this->getPeriodLastProcessedTimestamp($idSite);
656	+
657	+ Option::clearCachedOption($this->lastRunKey($idSite, "day"));
658	+ $lastTimestampWebsiteProcessedDay = $this->getDayLastProcessedTimestamp($idSite);
659	+ }
660	+
661	+ $this->updateIdSitesInvalidatedOldReports();
662	+
663	+ // For period other than days, we only re-process the reports at most
664	+ // 1) every $processPeriodsMaximumEverySeconds
665	+ $secondsSinceLastExecution = time() - $lastTimestampWebsiteProcessedPeriods;
666	+
667	+ // if timeout is more than 10 min, we account for a 5 min processing time, and allow trigger 1 min earlier
668	+ if ($this->processPeriodsMaximumEverySeconds > 10 * 60) {
669	+ $secondsSinceLastExecution += 5 * 60;
670	+ }
671	+
672	+ $shouldArchivePeriods = $secondsSinceLastExecution > $this->processPeriodsMaximumEverySeconds;
673	+ if (empty($lastTimestampWebsiteProcessedPeriods)) {
674	+ // 2) OR always if script never executed for this website before
675	+ $shouldArchivePeriods = true;
676	+ }
677	+
678	+ // (*) If the website is archived because it is a new day in its timezone
679	+ // We make sure all periods are archived, even if there is 0 visit today
680	+ $dayHasEndedMustReprocess = in_array($idSite, $this->websiteDayHasFinishedSinceLastRun);
681	+ if ($dayHasEndedMustReprocess) {
682	+ $shouldArchivePeriods = true;
683	+ }
684	+
685	+ // (*) If there was some old reports invalidated for this website
686	+ // we make sure all these old reports are triggered at least once
687	+ $websiteInvalidatedShouldReprocess = $this->isOldReportInvalidatedForWebsite($idSite);
688	+
689	+ if ($websiteInvalidatedShouldReprocess) {
690	+ $shouldArchivePeriods = true;
691	+ }
692	+
693	+ $websiteIdIsForced = in_array($idSite, $this->shouldArchiveSpecifiedSites);
694	+ if ($websiteIdIsForced) {
695	+ $shouldArchivePeriods = true;
696	+ }
697	+
698	+ // Test if we should process this website at all
699	+ $elapsedSinceLastArchiving = time() - $lastTimestampWebsiteProcessedDay;
700	+
701	+ // Skip this day archive if last archive was older than TTL
702	+ $existingArchiveIsValid = ($elapsedSinceLastArchiving < $this->todayArchiveTimeToLive);
703	+
704	+ $skipDayArchive = false;
705	+ if($existingArchiveIsValid
706	+ && !$websiteIdIsForced
707	+ && !$websiteInvalidatedShouldReprocess
708	+ && !$dayHasEndedMustReprocess
709	+ && $this->hasBeenProcessedSinceMidnight($idSite, $lastTimestampWebsiteProcessedDay)) {
710	+ $skipDayArchive = true;
711	+ }
712	+
713	+ if ($skipDayArchive) {
714	+ $this->logger->info("Skipped website id $idSite, already done "
715	+ . $this->formatter->getPrettyTimeFromSeconds($elapsedSinceLastArchiving, true)
716	+ . " ago, " . $timerWebsite->__toString());
717	+ $this->skippedDayArchivesWebsites++;
718	+ $this->skipped++;
719	+ return false;
720	+ }
721	+
722	+ /**
723	+ * Trigger archiving for days
724	+ */
725	+ try {
726	+ $shouldProceed = $this->processArchiveDays($idSite, $lastTimestampWebsiteProcessedDay, $shouldArchivePeriods, $timerWebsite);
727	+ } catch (UnexpectedWebsiteFoundException $e) {
728	+ // this website was deleted in the meantime
729	+ $shouldProceed = false;
730	+ $this->logger->info("Skipped website id $idSite, got: UnexpectedWebsiteFoundException, " . $timerWebsite->__toString());
731	+ }
732	+
733	+ if (!$shouldProceed) {
734	+ return false;
735	+ }
736	+
737	+ if (!$shouldArchivePeriods) {
738	+ $this->logger->info("Skipped website id $idSite periods processing, already done "
739	+ . $this->formatter->getPrettyTimeFromSeconds($elapsedSinceLastArchiving, true)
740	+ . " ago, " . $timerWebsite->__toString());
741	+ $this->skippedPeriodsArchivesWebsite++;
742	+ $this->skipped++;
743	+ return false;
744	+ }
745	+
746	+ /**
747	+ * Trigger archiving for non-day periods
748	+ */
749	+ try {
750	+ $success = $this->processArchiveForPeriods($idSite, $lastTimestampWebsiteProcessedPeriods);
751	+ } catch (UnexpectedWebsiteFoundException $e) {
752	+ // this website was deleted in the meantime
753	+ $this->logger->info("Skipped website id $idSite, got: UnexpectedWebsiteFoundException, " . $timerWebsite->__toString());
754	+ return false;
755	+ }
756	+
757	+ // Record successful run of this website's periods archiving
758	+ if ($success) {
759	+ Option::set($this->lastRunKey($idSite, "periods"), time());
760	+ }
761	+
762	+ if (!$success) {
763	+ // cancel marking the site as reprocessed
764	+ if ($websiteInvalidatedShouldReprocess) {
765	+ $store = new SitesToReprocessDistributedList();
766	+ $store->add($idSite);
767	+ }
768	+ }
769	+
770	+ $this->archivedPeriodsArchivesWebsite++;
771	+
772	+ $requestsWebsite = $this->requests - $requestsBefore;
773	+ $this->logger->info("Archived website id = $idSite, "
774	+ . $requestsWebsite . " API requests, "
775	+ . $timerWebsite->__toString()
776	+ . " [" . $this->websites->getNumProcessedWebsites() . "/"
777	+ . $this->websites->getNumSites()
778	+ . " done]");
779	+
780	+ return true;
781	+ }
782	+
783	+ /**
784	+ * @param $idSite
785	+ * @param $lastTimestampWebsiteProcessedPeriods
786	+ * @return bool
787	+ */
788	+ private function processArchiveForPeriods($idSite, $lastTimestampWebsiteProcessedPeriods)
789	+ {
790	+ $success = true;
791	+
792	+ foreach (array('week', 'month', 'year') as $period) {
793	+ if (!$this->shouldProcessPeriod($period)) {
794	+ // if any period was skipped, we do not mark the Periods archiving as successful
795	+ $success = false;
796	+ continue;
797	+ }
798	+
799	+ $timer = new Timer();
800	+
801	+ $date = $this->getApiDateParameter($idSite, $period, $lastTimestampWebsiteProcessedPeriods);
802	+ $periodArchiveWasSuccessful = $this->archiveReportsFor($idSite, $period, $date, $archiveSegments = true, $timer);
803	+ $success = $periodArchiveWasSuccessful && $success;
804	+ if(!$success) {
805	+ // if it failed, we abort the current website processing
806	+ return $success;
807	+ }
808	+ }
809	+
810	+ if ($this->shouldProcessPeriod('range')) {
811	+ // period=range
812	+ $customDateRangesToPreProcessForSite = $this->getCustomDateRangeToPreProcess($idSite);
813	+ foreach ($customDateRangesToPreProcessForSite as $dateRange) {
814	+ $timer = new Timer();
815	+ $archiveSegments = false; // do not pre-process segments for period=range #7611
816	+ $periodArchiveWasSuccessful = $this->archiveReportsFor($idSite, 'range', $dateRange, $archiveSegments, $timer);
817	+ $success = $periodArchiveWasSuccessful && $success;
818	+ }
819	+ }
820	+
821	+ return $success;
822	+ }
823	+
824	+ /**
825	+ * Returns base URL to process reports for the $idSite on a given $period
826	+ *
827	+ * @param string $idSite
828	+ * @param string $period
829	+ * @param string $date
830	+ * @param bool\|false $segment
831	+ * @return string
832	+ */
833	+ private function getVisitsRequestUrl($idSite, $period, $date, $segment = false)
834	+ {
835	+ $request = "?module=API&method=API.get&idSite=$idSite&period=$period&date=" . $date . "&format=php";
836	+ if ($segment) {
837	+ $request .= '&segment=' . urlencode($segment);
838	+ }
839	+ return $request;
840	+ }
841	+
842	+ private function initSegmentsToArchive()
843	+ {
844	+ $segments = \Piwik\SettingsPiwik::getKnownSegmentsToArchive();
845	+
846	+ if (empty($segments)) {
847	+ return array();
848	+ }
849	+
850	+ $this->logger->info("- Will pre-process " . count($segments) . " Segments for each website and each period: " . implode(", ", $segments));
851	+ return $segments;
852	+ }
853	+
854	+ /**
855	+ * @param $idSite
856	+ * @param $lastTimestampWebsiteProcessedDay
857	+ * @param $shouldArchivePeriods
858	+ * @param $timerWebsite
859	+ * @return bool
860	+ */
861	+ protected function processArchiveDays($idSite, $lastTimestampWebsiteProcessedDay, $shouldArchivePeriods, Timer $timerWebsite)
862	+ {
863	+ if (!$this->shouldProcessPeriod("day")) {
864	+ // skip day archiving and proceed to period processing
865	+ return true;
866	+ }
867	+
868	+ $timer = new Timer();
869	+
870	+ // Remove this website from the list of websites to be invalidated
871	+ // since it's now just about to being re-processed, makes sure another running cron archiving process
872	+ // does not archive the same idSite
873	+ $websiteInvalidatedShouldReprocess = $this->isOldReportInvalidatedForWebsite($idSite);
874	+ if ($websiteInvalidatedShouldReprocess) {
875	+ $store = new SitesToReprocessDistributedList();
876	+ $store->remove($idSite);
877	+ }
878	+
879	+ // when some data was purged from this website
880	+ // we make sure we query all previous days/weeks/months
881	+ $processDaysSince = $lastTimestampWebsiteProcessedDay;
882	+ if ($websiteInvalidatedShouldReprocess
883	+ // when --force-all-websites option,
884	+ // also forces to archive last52 days to be safe
885	+ \|\| $this->shouldArchiveAllSites) {
886	+ $processDaysSince = false;
887	+ }
888	+
889	+ $date = $this->getApiDateParameter($idSite, "day", $processDaysSince);
890	+ $url = $this->getVisitsRequestUrl($idSite, "day", $date);
891	+
892	+ $cliMulti = $this->makeCliMulti();
893	+ if ($cliMulti->isCommandAlreadyRunning($this->makeRequestUrl($url))) {
894	+ $this->logger->info("Skipped website id $idSite, such a process is already in progress, " . $timerWebsite->__toString());
895	+ $this->skipped++;
896	+ return false;
897	+ }
898	+
899	+ $visitsLastDays = 0;
900	+
901	+ list($isThereArchive, $newDate) = $this->isThereAValidArchiveForPeriod($idSite, 'day', $date, $segment = '');
902	+ if ($isThereArchive) {
903	+ $visitsToday = Archive::build($idSite, 'day', $date)->getNumeric('nb_visits');
904	+ $visitsToday = end($visitsToday);
905	+ $visitsToday = isset($visitsToday['nb_visits']) ? $visitsToday['nb_visits'] : 0;
906	+
907	+ $this->logArchiveWebsiteSkippedValidArchiveExists($idSite, 'day', $date);
908	+ ++$this->skipped;
909	+ } else {
910	+ $date = $newDate; // use modified lastN param
911	+
912	+ $this->logArchiveWebsite($idSite, "day", $date);
913	+
914	+ $content = $this->request($url);
915	+ $daysResponse = Common::safe_unserialize($content);
916	+
917	+ if (empty($content)
918	+ \|\| !is_array($daysResponse)
919	+ \|\| count($daysResponse) == 0
920	+ ) {
921	+ // cancel marking the site as reprocessed
922	+ if ($websiteInvalidatedShouldReprocess) {
923	+ $store = new SitesToReprocessDistributedList();
924	+ $store->add($idSite);
925	+ }
926	+
927	+ $this->logError("Empty or invalid response '$content' for website id $idSite, " . $timerWebsite->__toString() . ", skipping");
928	+ $this->skippedDayOnApiError++;
929	+ $this->skipped++;
930	+ return false;
931	+ }
932	+
933	+ $visitsToday = $this->getVisitsLastPeriodFromApiResponse($daysResponse);
934	+ $visitsLastDays = $this->getVisitsFromApiResponse($daysResponse);
935	+
936	+ $this->requests++;
937	+ $this->processed++;
938	+
939	+ $shouldArchiveWithoutVisits = PluginsArchiver::doesAnyPluginArchiveWithoutVisits();
940	+
941	+ // If there is no visit today and we don't need to process this website, we can skip remaining archives
942	+ if (
943	+ 0 == $visitsToday && !$shouldArchiveWithoutVisits
944	+ && !$shouldArchivePeriods
945	+ ) {
946	+ $this->logger->info("Skipped website id $idSite, no visit today, " . $timerWebsite->__toString());
947	+ $this->skippedDayNoRecentData++;
948	+ $this->skipped++;
949	+ return false;
950	+ }
951	+
952	+ if (0 == $visitsLastDays && !$shouldArchiveWithoutVisits
953	+ && !$shouldArchivePeriods
954	+ && $this->shouldArchiveAllSites
955	+ ) {
956	+ $humanReadableDate = $this->formatReadableDateRange($date);
957	+ $this->logger->info("Skipped website id $idSite, no visits in the $humanReadableDate days, " . $timerWebsite->__toString());
958	+ $this->skippedPeriodsNoDataInPeriod++;
959	+ $this->skipped++;
960	+ return false;
961	+ }
962	+ }
963	+
964	+ $this->visitsToday += $visitsToday;
965	+ $this->websitesWithVisitsSinceLastRun++;
966	+
967	+ $dayArchiveWasSuccessful = $this->archiveReportsFor($idSite, "day", $this->getApiDateParameter($idSite, "day", $processDaysSince), $archiveSegments = true, $timer, $visitsToday, $visitsLastDays);
968	+
969	+ if($dayArchiveWasSuccessful) {
970	+ Option::set($this->lastRunKey($idSite, "day"), time());
971	+ }
972	+ return $dayArchiveWasSuccessful;
973	+ }
974	+
975	+ private function isThereAValidArchiveForPeriod($idSite, $period, $date, $segment = '')
976	+ {
977	+ if (Range::isMultiplePeriod($date, $period)) {
978	+ $rangePeriod = Factory::build($period, $date, Site::getTimezoneFor($idSite));
979	+ $periodsToCheck = $rangePeriod->getSubperiods();
980	+ } else {
981	+ $periodsToCheck = [Factory::build($period, $date, Site::getTimezoneFor($idSite))];
982	+ }
983	+
984	+ $periodsToCheckRanges = array_map(function (Period $p) { return $p->getRangeString(); }, $periodsToCheck);
985	+
986	+ $this->invalidateArchivedReportsForSitesThatNeedToBeArchivedAgain();
987	+
988	+ $archiveIds = ArchiveSelector::getArchiveIds(
989	+ [$idSite], $periodsToCheck, new Segment($segment, [$idSite]), $plugins = [], // empty plugins param since we only check for an 'all' archive
990	+ $includeInvalidated = false
991	+ );
992	+
993	+ $foundArchivePeriods = [];
994	+ foreach ($archiveIds as $doneFlag => $dates) {
995	+ foreach ($dates as $dateRange => $idArchives) {
996	+ $foundArchivePeriods[] = $dateRange;
997	+ }
998	+ }
999	+
1000	+ $diff = array_diff($periodsToCheckRanges, $foundArchivePeriods);
1001	+ $isThereArchiveForAllPeriods = empty($diff);
1002	+
1003	+ // if there is an invalidated archive within the range, find out the oldest one and how far it is from today,
1004	+ // and change the lastN $date to be value so it is correctly re-processed.
1005	+ $newDate = $date;
1006	+ if (!$isThereArchiveForAllPeriods
1007	+ && preg_match('/^last([0-9]+)/', $date, $matches)
1008	+ ) {
1009	+ $lastNValue = (int) $matches[1];
1010	+
1011	+ usort($diff, function ($lhs, $rhs) {
1012	+ $lhsDate = explode(',', $lhs)[0];
1013	+ $rhsDate = explode(',', $rhs)[0];
1014	+
1015	+ if ($lhsDate == $rhsDate) {
1016	+ return 1;
1017	+ } else if (Date::factory($lhsDate)->isEarlier(Date::factory($rhsDate))) {
1018	+ return -1;
1019	+ } else {
1020	+ return 1;
1021	+ }
1022	+ });
1023	+
1024	+ $oldestDateWithoutArchive = explode(',', reset($diff))[0];
1025	+ $todayInTimezone = Date::factoryInTimezone('today', Site::getTimezoneFor($idSite));
1026	+
1027	+ /** @var Range $newRangePeriod */
1028	+ $newRangePeriod = PeriodFactory::build($period, $oldestDateWithoutArchive . ',' . $todayInTimezone);
1029	+
1030	+ $newDate = 'last' . min($lastNValue, $newRangePeriod->getNumberOfSubperiods());
1031	+ }
1032	+
1033	+ return [$isThereArchiveForAllPeriods, $newDate];
1034	+ }
1035	+
1036	+ /**
1037	+ * @param $idSite
1038	+ * @return array
1039	+ */
1040	+ private function getSegmentsForSite($idSite)
1041	+ {
1042	+ $segmentsAllSites = $this->segments;
1043	+ $segmentsThisSite = SettingsPiwik::getKnownSegmentsToArchiveForSite($idSite);
1044	+ $segments = array_unique(array_merge($segmentsAllSites, $segmentsThisSite));
1045	+ return $segments;
1046	+ }
1047	+
1048	+ private function formatReadableDateRange($date)
1049	+ {
1050	+ if (0 === strpos($date, 'last')) {
1051	+ $readable = 'last ' . str_replace('last', '', $date);
1052	+ } elseif (0 === strpos($date, 'previous')) {
1053	+ $readable = 'previous ' . str_replace('previous', '', $date);
1054	+ } else {
1055	+ $readable = 'last ' . $date;
1056	+ }
1057	+
1058	+ return $readable;
1059	+ }
1060	+
1061	+ /**
1062	+ * Will trigger API requests for the specified Website $idSite,
1063	+ * for the specified $period, for all segments that are pre-processed for this website.
1064	+ * Requests are triggered using cURL multi handle
1065	+ *
1066	+ * @param $idSite int
1067	+ * @param $period string
1068	+ * @param $date string
1069	+ * @param $archiveSegments bool Whether to pre-process all custom segments
1070	+ * @param Timer $periodTimer
1071	+ * @param $visitsToday int Visits for the "day" period of today
1072	+ * @param $visitsLastDays int Visits for the last N days periods
1073	+ * @return bool True on success, false if some request failed
1074	+ */
1075	+ private function archiveReportsFor($idSite, $period, $date, $archiveSegments, Timer $periodTimer, $visitsToday = 0, $visitsLastDays = 0)
1076	+ {
1077	+ $url = $this->getVisitsRequestUrl($idSite, $period, $date, $segment = false);
1078	+ $url = $this->makeRequestUrl($url);
1079	+
1080	+ $visitsInLastPeriod = $visitsToday;
1081	+ $visitsInLastPeriods = $visitsLastDays;
1082	+ $success = true;
1083	+
1084	+ $urls = array();
1085	+
1086	+ $cliMulti = $this->makeCliMulti();
1087	+
1088	+ $noSegmentUrl = $url;
1089	+
1090	+ // already processed above for "day"
1091	+ if ($period != "day") {
1092	+
1093	+ if ($this->isAlreadyArchivingUrl($url, $idSite, $period, $date)) {
1094	+ $success = false;
1095	+ return $success;
1096	+ }
1097	+
1098	+ $self = $this;
1099	+ $request = new Request($url);
1100	+ $request->before(function () use ($self, $url, $idSite, $period, $date, $segment, $request) {
1101	+ if ($self->isAlreadyArchivingUrl($url, $idSite, $period, $date)) {
1102	+ return Request::ABORT;
1103	+ }
1104	+
1105	+ list($isThereArchive, $newDate) = $this->isThereAValidArchiveForPeriod($idSite, $period, $date, $segment);
1106	+ if ($isThereArchive) {
1107	+ $this->logArchiveWebsiteSkippedValidArchiveExists($idSite, $period, $date);
1108	+ return Request::ABORT;
1109	+ }
1110	+
1111	+ $urlBefore = $request->getUrl();
1112	+ $request->changeDate($newDate);
1113	+ $request->makeSureDateIsNotSingleDayRange();
1114	+
1115	+ // check again if we are already archiving the URL since we just changed it
1116	+ if ($request->getUrl() !== $urlBefore
1117	+ && $self->isAlreadyArchivingSegment($request->getUrl(), $idSite, $period, $segment)
1118	+ ) {
1119	+ return Request::ABORT;
1120	+ }
1121	+
1122	+ $this->logArchiveWebsite($idSite, $period, $newDate);
1123	+ });
1124	+ $urls[] = $request;
1125	+ }
1126	+
1127	+ $segmentRequestsCount = 0;
1128	+ if ($archiveSegments) {
1129	+ $urlsWithSegment = $this->getUrlsWithSegment($idSite, $period, $date);
1130	+ $urls = array_merge($urls, $urlsWithSegment);
1131	+ $segmentRequestsCount = count($urlsWithSegment);
1132	+
1133	+ // in case several segment URLs for period=range had the date= rewritten to the same value, we only call API once
1134	+ $urls = array_unique($urls);
1135	+ }
1136	+
1137	+ $this->requests += count($urls);
1138	+
1139	+ $response = $cliMulti->request($urls);
1140	+
1141	+ foreach ($urls as $index => $url) {
1142	+ $content = array_key_exists($index, $response) ? $response[$index] : null;
1143	+ $success = $success && $this->checkResponse($content, $url);
1144	+
1145	+ if ($noSegmentUrl == $url && $success) {
1146	+ $stats = Common::safe_unserialize($content);
1147	+
1148	+ if (!is_array($stats)) {
1149	+ $this->logError("Error unserializing the following response from $url: " . $content);
1150	+ $success = false;
1151	+ }
1152	+
1153	+ if ($period == 'range') {
1154	+ // range returns one dataset (the sum of data between the two dates),
1155	+ // whereas other periods return lastN which is N datasets in an array. Here we make our period=range dataset look like others:
1156	+ $stats = array($stats);
1157	+ }
1158	+
1159	+ $visitsInLastPeriods = $this->getVisitsFromApiResponse($stats);
1160	+ $visitsInLastPeriod = $this->getVisitsLastPeriodFromApiResponse($stats);
1161	+ }
1162	+ }
1163	+
1164	+ $this->logArchivedWebsite($idSite, $period, $date, $segmentRequestsCount, $visitsInLastPeriods, $visitsInLastPeriod, $periodTimer);
1165	+
1166	+ return $success;
1167	+ }
1168	+
1169	+ // TODO: need test to make sure segment archives are invalidated as well
1170	+ private function logArchiveWebsiteSkippedValidArchiveExists($idSite, $period, $date, $segment = '')
1171	+ {
1172	+ $this->logger->info("Skipping archiving for website id = {idSite}, period = {period}, date = {date}, segment = {segment}, "
1173	+ . "since there is already a valid archive (tracking a visit automatically invalidates archives).", [
1174	+ 'idSite' => $idSite,
1175	+ 'period' => $period,
1176	+ 'date' => $date,
1177	+ 'segment' => $segment,
1178	+ ]);
1179	+ }
1180	+
1181	+ /**
1182	+ * Logs a section in the output
1183	+ *
1184	+ * @param string $title
1185	+ */
1186	+ private function logSection($title = "")
1187	+ {
1188	+ $this->logger->info("---------------------------");
1189	+ if (!empty($title)) {
1190	+ $this->logger->info($title);
1191	+ }
1192	+ }
1193	+
1194	+ public function logError($m)
1195	+ {
1196	+ if (!defined('PIWIK_ARCHIVE_NO_TRUNCATE')) {
1197	+ $m = substr($m, 0, self::TRUNCATE_ERROR_MESSAGE_SUMMARY);
1198	+ $m = str_replace(array("\n", "\t"), " ", $m);
1199	+ }
1200	+ $this->errors[] = $m;
1201	+ $this->logger->error($m);
1202	+ }
1203	+
1204	+ private function logNetworkError($url, $response)
1205	+ {
1206	+ $message = "Got invalid response from API request: $url. ";
1207	+ if (empty($response)) {
1208	+ $message .= "The response was empty. This usually means a server error. A solution to this error is generally to increase the value of 'memory_limit' in your php.ini file. ";
1209	+
1210	+ if($this->makeCliMulti()->supportsAsync()) {
1211	+ $message .= " For more information and the error message please check in your PHP CLI error log file. As this core:archive command triggers PHP processes over the CLI, you can find where PHP CLI logs are stored by running this command: php -i \| grep error_log";
1212	+ } else {
1213	+ $message .= " For more information and the error message please check your web server's error Log file. As this core:archive command triggers PHP processes over HTTP, you can find the error message in your Matomo's web server error logs. ";
1214	+ }
1215	+ } else {
1216	+ $message .= "Response was '$response'";
1217	+ }
1218	+
1219	+ $this->logError($message);
1220	+ return false;
1221	+ }
1222	+
1223	+ /**
1224	+ * Issues a request to $url eg. "?module=API&method=API.getDefaultMetricTranslations&format=original&serialize=1"
1225	+ *
1226	+ * @param string $url
1227	+ * @return string
1228	+ */
1229	+ private function request($url)
1230	+ {
1231	+ $url = $this->makeRequestUrl($url);
1232	+
1233	+ try {
1234	+ $cliMulti = $this->makeCliMulti();
1235	+ $responses = $cliMulti->request(array($url));
1236	+
1237	+ $response = !empty($responses) ? array_shift($responses) : null;
1238	+ } catch (Exception $e) {
1239	+ return $this->logNetworkError($url, $e->getMessage());
1240	+ }
1241	+ if ($this->checkResponse($response, $url)) {
1242	+ return $response;
1243	+ }
1244	+ return false;
1245	+ }
1246	+
1247	+ private function checkResponse($response, $url)
1248	+ {
1249	+ if (empty($response)
1250	+ \|\| stripos($response, 'error') !== false
1251	+ ) {
1252	+ return $this->logNetworkError($url, $response);
1253	+ }
1254	+ return true;
1255	+ }
1256	+
1257	+ /**
1258	+ * Initializes the various parameters to the script, based on input parameters.
1259	+ *
1260	+ */
1261	+ private function initStateFromParameters()
1262	+ {
1263	+ $this->todayArchiveTimeToLive = Rules::getTodayArchiveTimeToLive();
1264	+ $this->processPeriodsMaximumEverySeconds = $this->getDelayBetweenPeriodsArchives();
1265	+ $this->lastSuccessRunTimestamp = $this->getLastSuccessRunTimestamp();
1266	+ $this->shouldArchiveOnlySitesWithTrafficSince = $this->isShouldArchiveAllSitesWithTrafficSince();
1267	+ $this->shouldArchiveOnlySpecificPeriods = $this->getPeriodsToProcess();
1268	+
1269	+ if ($this->shouldArchiveOnlySitesWithTrafficSince !== false) {
1270	+ // force-all-periods is set here
1271	+ $this->archiveAndRespectTTL = false;
1272	+ }
1273	+ }
1274	+
1275	+ private function getSecondsSinceLastArchive()
1276	+ {
1277	+ $wasNotCustomTimeRequested = $this->shouldArchiveOnlySitesWithTrafficSince === false;
1278	+
1279	+ if ($wasNotCustomTimeRequested && !empty($this->lastSuccessRunTimestamp)) {
1280	+ // there was a previous successful run
1281	+
1282	+ return time() - $this->lastSuccessRunTimestamp;
1283	+
1284	+ } elseif (is_numeric($this->shouldArchiveOnlySitesWithTrafficSince)) {
1285	+ // $shouldArchiveAllPeriodsSince was specified
1286	+ $secondsSinceStart = time() - $this->archivingStartingTime;
1287	+ return $this->shouldArchiveOnlySitesWithTrafficSince + $secondsSinceStart;
1288	+ }
1289	+
1290	+ // force-all-periods without value
1291	+ return self::ARCHIVE_SITES_WITH_TRAFFIC_SINCE;
1292	+ }
1293	+
1294	+ public function filterWebsiteIds(&$websiteIds)
1295	+ {
1296	+ // Keep only the websites that do exist
1297	+ $websiteIds = array_intersect($websiteIds, $this->allWebsites);
1298	+
1299	+ /**
1300	+ * Triggered by the core:archive console command so plugins can modify the list of
1301	+ * websites that the archiving process will be launched for.
1302	+ *
1303	+ * Plugins can use this hook to add websites to archive, remove websites to archive, or change
1304	+ * the order in which websites will be archived.
1305	+ *
1306	+ * @param array $websiteIds The list of website IDs to launch the archiving process for.
1307	+ */
1308	+ Piwik::postEvent('CronArchive.filterWebsiteIds', array(&$websiteIds));
1309	+ }
1310	+
1311	+ /**
1312	+ * @internal
1313	+ * @param $api
1314	+ */
1315	+ public function setApiToInvalidateArchivedReport($api)
1316	+ {
1317	+ $this->apiToInvalidateArchivedReport = $api;
1318	+ }
1319	+
1320	+ private function getApiToInvalidateArchivedReport()
1321	+ {
1322	+ if ($this->apiToInvalidateArchivedReport) {
1323	+ return $this->apiToInvalidateArchivedReport;
1324	+ }
1325	+
1326	+ return CoreAdminHomeAPI::getInstance();
1327	+ }
1328	+
1329	+ public function invalidateArchivedReportsForSitesThatNeedToBeArchivedAgain()
1330	+ {
1331	+ $sitesPerDays = $this->invalidator->getRememberedArchivedReportsThatShouldBeInvalidated();
1332	+
1333	+ foreach ($sitesPerDays as $date => $siteIds) {
1334	+ //Concurrent transaction logic will end up with duplicates set. Adding array_unique to the siteIds.
1335	+ $listSiteIds = implode(',', array_unique($siteIds ));
1336	+
1337	+ try {
1338	+ $this->logger->info('- Will invalidate archived reports for ' . $date . ' for following websites ids: ' . $listSiteIds);
1339	+ $this->getApiToInvalidateArchivedReport()->invalidateArchivedReports($siteIds, $date);
1340	+ } catch (Exception $e) {
1341	+ $this->logger->info('Failed to invalidate archived reports: ' . $e->getMessage());
1342	+ }
1343	+ }
1344	+ }
1345	+
1346	+ /**
1347	+ * Returns the list of sites to loop over and archive.
1348	+ * @return array
1349	+ */
1350	+ public function initWebsiteIds()
1351	+ {
1352	+ if (count($this->shouldArchiveSpecifiedSites) > 0) {
1353	+ $this->logger->info("- Will process " . count($this->shouldArchiveSpecifiedSites) . " websites (--force-idsites)");
1354	+
1355	+ return $this->shouldArchiveSpecifiedSites;
1356	+ }
1357	+
1358	+ $this->findWebsiteIdsInTimezoneWithNewDay($this->allWebsites);
1359	+ $this->findInvalidatedSitesToReprocess();
1360	+
1361	+ if ($this->shouldArchiveAllSites) {
1362	+ $this->logger->info("- Will process all " . count($this->allWebsites) . " websites");
1363	+ }
1364	+
1365	+ return $this->allWebsites;
1366	+ }
1367	+
1368	+ private function updateIdSitesInvalidatedOldReports()
1369	+ {
1370	+ $store = new SitesToReprocessDistributedList();
1371	+ $this->idSitesInvalidatedOldReports = $store->getAll();
1372	+ }
1373	+
1374	+ /**
1375	+ * Return All websites that had reports in the past which were invalidated recently
1376	+ * (see API CoreAdminHome.invalidateArchivedReports)
1377	+ * eg. when using Python log import script
1378	+ *
1379	+ * @return array
1380	+ */
1381	+ private function findInvalidatedSitesToReprocess()
1382	+ {
1383	+ $this->updateIdSitesInvalidatedOldReports();
1384	+
1385	+ if (count($this->idSitesInvalidatedOldReports) > 0) {
1386	+ $ids = ", IDs: " . implode(", ", $this->idSitesInvalidatedOldReports);
1387	+ $this->logger->info("- Will process " . count($this->idSitesInvalidatedOldReports)
1388	+ . " other websites because some old data reports have been invalidated (eg. using the Log Import script or the InvalidateReports plugin) "
1389	+ . $ids);
1390	+ }
1391	+
1392	+ return $this->idSitesInvalidatedOldReports;
1393	+ }
1394	+
1395	+ /**
1396	+ * Detects whether a site had visits since midnight in the websites timezone
1397	+ *
1398	+ * @param $idSite
1399	+ * @return bool
1400	+ */
1401	+ private function hadWebsiteTrafficSinceMidnightInTimezone($idSite)
1402	+ {
1403	+ $timezone = Site::getTimezoneFor($idSite);
1404	+
1405	+ $nowInTimezone = Date::factoryInTimezone('now', $timezone);
1406	+ $midnightInTimezone = $nowInTimezone->setTime('00:00:00');
1407	+
1408	+ $secondsSinceMidnight = $nowInTimezone->getTimestamp() - $midnightInTimezone->getTimestamp();
1409	+
1410	+ $secondsSinceLastArchive = $this->getSecondsSinceLastArchive();
1411	+ if ($secondsSinceLastArchive < $secondsSinceMidnight) {
1412	+ $secondsBackToLookForVisits = $secondsSinceLastArchive;
1413	+ $sinceInfo = "(since the last successful archiving)";
1414	+ } else {
1415	+ $secondsBackToLookForVisits = $secondsSinceMidnight;
1416	+ $sinceInfo = "(since midnight)";
1417	+ }
1418	+
1419	+ $from = Date::now()->subSeconds($secondsBackToLookForVisits)->getDatetime();
1420	+ $to = Date::now()->addHour(1)->getDatetime();
1421	+
1422	+ $dao = new RawLogDao();
1423	+ $hasVisits = $dao->hasSiteVisitsBetweenTimeframe($from, $to, $idSite);
1424	+
1425	+ if ($hasVisits) {
1426	+ $this->logger->info("- tracking data found for website id $idSite since $from UTC $sinceInfo");
1427	+ } else {
1428	+ $this->logger->info("- no new tracking data for website id $idSite since $from UTC $sinceInfo");
1429	+ }
1430	+
1431	+ return $hasVisits;
1432	+ }
1433	+
1434	+ /**
1435	+ * Returns the list of timezones where the specified timestamp in that timezone
1436	+ * is on a different day than today in that timezone.
1437	+ *
1438	+ * @return array
1439	+ */
1440	+ private function getTimezonesHavingNewDaySinceLastRun()
1441	+ {
1442	+ $timestamp = $this->lastSuccessRunTimestamp;
1443	+ $uniqueTimezones = APISitesManager::getInstance()->getUniqueSiteTimezones();
1444	+ $timezoneToProcess = array();
1445	+ foreach ($uniqueTimezones as &$timezone) {
1446	+ $processedDateInTz = Date::factory((int)$timestamp, $timezone);
1447	+ $currentDateInTz = Date::factory('now', $timezone);
1448	+
1449	+ if ($processedDateInTz->toString() != $currentDateInTz->toString()) {
1450	+ $timezoneToProcess[] = $timezone;
1451	+ }
1452	+ }
1453	+ return $timezoneToProcess;
1454	+ }
1455	+
1456	+ private function hasBeenProcessedSinceMidnight($idSite, $lastTimestampWebsiteProcessedDay)
1457	+ {
1458	+ if (false === $lastTimestampWebsiteProcessedDay) {
1459	+ return true;
1460	+ }
1461	+
1462	+ $timezone = Site::getTimezoneFor($idSite);
1463	+
1464	+ $dateInTimezone = Date::factory('now', $timezone);
1465	+ $midnightInTimezone = $dateInTimezone->setTime('00:00:00');
1466	+
1467	+ $lastProcessedDateInTimezone = Date::factory((int) $lastTimestampWebsiteProcessedDay, $timezone);
1468	+
1469	+ return $lastProcessedDateInTimezone->getTimestamp() >= $midnightInTimezone->getTimestamp();
1470	+ }
1471	+
1472	+ /**
1473	+ * Returns the list of websites in which timezones today is a new day
1474	+ * (compared to the last time archiving was executed)
1475	+ *
1476	+ * @param $websiteIds
1477	+ * @return array Website IDs
1478	+ */
1479	+ private function findWebsiteIdsInTimezoneWithNewDay($websiteIds)
1480	+ {
1481	+ $timezones = $this->getTimezonesHavingNewDaySinceLastRun();
1482	+ $websiteDayHasFinishedSinceLastRun = APISitesManager::getInstance()->getSitesIdFromTimezones($timezones);
1483	+ $websiteDayHasFinishedSinceLastRun = array_intersect($websiteDayHasFinishedSinceLastRun, $websiteIds);
1484	+ $this->websiteDayHasFinishedSinceLastRun = $websiteDayHasFinishedSinceLastRun;
1485	+
1486	+ if (count($websiteDayHasFinishedSinceLastRun) > 0) {
1487	+ $ids = !empty($websiteDayHasFinishedSinceLastRun) ? ", IDs: " . implode(", ", $websiteDayHasFinishedSinceLastRun) : "";
1488	+ $this->logger->info("- Will process " . count($websiteDayHasFinishedSinceLastRun)
1489	+ . " other websites because the last time they were archived was on a different day (in the website's timezone) "
1490	+ . $ids);
1491	+ }
1492	+
1493	+ return $websiteDayHasFinishedSinceLastRun;
1494	+ }
1495	+
1496	+ private function logInitInfo()
1497	+ {
1498	+ $this->logSection("INIT");
1499	+ $this->logger->info("Running Matomo " . Version::VERSION . " as Super User");
1500	+ }
1501	+
1502	+ private function logArchiveTimeoutInfo()
1503	+ {
1504	+ $this->logSection("NOTES");
1505	+
1506	+ // Recommend to disable browser archiving when using this script
1507	+ if (Rules::isBrowserTriggerEnabled()) {
1508	+ $this->logger->info("- If you execute this script at least once per hour (or more often) in a crontab, you may disable 'Browser trigger archiving' in Matomo UI > Settings > General Settings.");
1509	+ $this->logger->info(" See the doc at: https://matomo.org/docs/setup-auto-archiving/");
1510	+ }
1511	+
1512	+ $cliMulti = new CliMulti();
1513	+ $supportsAsync = $cliMulti->supportsAsync();
1514	+ $this->logger->info("- " . ($supportsAsync ? 'Async process archiving supported, using CliMulti.' : 'Async process archiving not supported, using curl requests.'));
1515	+
1516	+ $this->logger->info("- Reports for today will be processed at most every " . $this->todayArchiveTimeToLive
1517	+ . " seconds. You can change this value in Matomo UI > Settings > General Settings.");
1518	+
1519	+ $this->logger->info("- Reports for the current week/month/year will be requested at most every "
1520	+ . $this->processPeriodsMaximumEverySeconds . " seconds.");
1521	+
1522	+ foreach (array('week', 'month', 'year', 'range') as $period) {
1523	+ $ttl = Rules::getPeriodArchiveTimeToLiveDefault($period);
1524	+
1525	+ if (!empty($ttl) && $ttl !== $this->todayArchiveTimeToLive) {
1526	+ $this->logger->info("- Reports for the current $period will be processed at most every " . $ttl
1527	+ . " seconds. You can change this value in config/config.ini.php by editing 'time_before_" . $period . "_archive_considered_outdated' in the '[General]' section.");
1528	+ }
1529	+ }
1530	+
1531	+ // Try and not request older data we know is already archived
1532	+ if ($this->lastSuccessRunTimestamp !== false) {
1533	+ $dateLast = time() - $this->lastSuccessRunTimestamp;
1534	+ $this->logger->info("- Archiving was last executed without error "
1535	+ . $this->formatter->getPrettyTimeFromSeconds($dateLast, true) . " ago");
1536	+ }
1537	+ }
1538	+
1539	+ /**
1540	+ * Returns the delay in seconds, that should be enforced, between calling archiving for Periods Archives.
1541	+ * It can be set by --force-timeout-for-periods=X
1542	+ *
1543	+ * @return int
1544	+ */
1545	+ private function getDelayBetweenPeriodsArchives()
1546	+ {
1547	+ if (empty($this->forceTimeoutPeriod)) {
1548	+ return self::SECONDS_DELAY_BETWEEN_PERIOD_ARCHIVES;
1549	+ }
1550	+
1551	+ // Ensure the cache for periods is at least as high as cache for today
1552	+ if ($this->forceTimeoutPeriod > $this->todayArchiveTimeToLive) {
1553	+ return $this->forceTimeoutPeriod;
1554	+ }
1555	+
1556	+ $this->logger->info("WARNING: Automatically increasing --force-timeout-for-periods from {$this->forceTimeoutPeriod} to "
1557	+ . $this->todayArchiveTimeToLive
1558	+ . " to match the cache timeout for Today's report specified in Matomo UI > Settings > General Settings");
1559	+
1560	+ return $this->todayArchiveTimeToLive;
1561	+ }
1562	+
1563	+ private function isShouldArchiveAllSitesWithTrafficSince()
1564	+ {
1565	+ if (empty($this->shouldArchiveAllPeriodsSince)) {
1566	+ return false;
1567	+ }
1568	+
1569	+ if (is_numeric($this->shouldArchiveAllPeriodsSince)
1570	+ && $this->shouldArchiveAllPeriodsSince > 1
1571	+ ) {
1572	+ return (int)$this->shouldArchiveAllPeriodsSince;
1573	+ }
1574	+
1575	+ return true;
1576	+ }
1577	+
1578	+ private function getVisitsLastPeriodFromApiResponse($stats)
1579	+ {
1580	+ if (empty($stats)) {
1581	+ return 0;
1582	+ }
1583	+
1584	+ $today = end($stats);
1585	+
1586	+ if (empty($today['nb_visits'])) {
1587	+ return 0;
1588	+ }
1589	+
1590	+ return $today['nb_visits'];
1591	+ }
1592	+
1593	+ private function getVisitsFromApiResponse($stats)
1594	+ {
1595	+ if (empty($stats)) {
1596	+ return 0;
1597	+ }
1598	+
1599	+ $visits = 0;
1600	+ foreach ($stats as $metrics) {
1601	+ if (empty($metrics['nb_visits'])) {
1602	+ continue;
1603	+ }
1604	+ $visits += $metrics['nb_visits'];
1605	+ }
1606	+
1607	+ return $visits;
1608	+ }
1609	+
1610	+ /**
1611	+ * @param $idSite
1612	+ * @param $period
1613	+ * @param $lastTimestampWebsiteProcessed
1614	+ * @return float\|int\|true
1615	+ */
1616	+ private function getApiDateParameter($idSite, $period, $lastTimestampWebsiteProcessed = false)
1617	+ {
1618	+ $dateRangeForced = $this->getDateRangeToProcess();
1619	+
1620	+ if (!empty($dateRangeForced)) {
1621	+ return $dateRangeForced;
1622	+ }
1623	+
1624	+ return $this->getDateLastN($idSite, $period, $lastTimestampWebsiteProcessed);
1625	+ }
1626	+
1627	+ /**
1628	+ * @param $idSite
1629	+ * @param $period
1630	+ * @param $date
1631	+ * @param $segmentsCount
1632	+ * @param $visitsInLastPeriods
1633	+ * @param $visitsToday
1634	+ * @param $timer
1635	+ */
1636	+ private function logArchivedWebsite($idSite, $period, $date, $segmentsCount, $visitsInLastPeriods, $visitsToday, Timer $timer)
1637	+ {
1638	+ if (strpos($date, 'last') === 0 \|\| strpos($date, 'previous') === 0) {
1639	+ $humanReadable = $this->formatReadableDateRange($date);
1640	+ $visitsInLastPeriods = (int)$visitsInLastPeriods . " visits in $humanReadable " . $period . "s, ";
1641	+ $thisPeriod = $period == "day" ? "today" : "this " . $period;
1642	+ $visitsInLastPeriod = (int)$visitsToday . " visits " . $thisPeriod . ", ";
1643	+ } else {
1644	+ $visitsInLastPeriods = (int)$visitsInLastPeriods . " visits in " . $period . "s included in: $date, ";
1645	+ $visitsInLastPeriod = '';
1646	+ }
1647	+
1648	+ $this->logger->info("Archived website id = $idSite, period = $period, $segmentsCount segments, "
1649	+ . $visitsInLastPeriods
1650	+ . $visitsInLastPeriod
1651	+ . $timer->__toString());
1652	+ }
1653	+
1654	+ private function getDateRangeToProcess()
1655	+ {
1656	+ if (empty($this->restrictToDateRange)) {
1657	+ return false;
1658	+ }
1659	+
1660	+ if (strpos($this->restrictToDateRange, ',') === false) {
1661	+ throw new Exception("--force-date-range expects a date range ie. YYYY-MM-DD,YYYY-MM-DD");
1662	+ }
1663	+
1664	+ return $this->restrictToDateRange;
1665	+ }
1666	+
1667	+ /**
1668	+ * @return array
1669	+ */
1670	+ private function getPeriodsToProcess()
1671	+ {
1672	+ $this->restrictToPeriods = array_intersect($this->restrictToPeriods, $this->getDefaultPeriodsToProcess());
1673	+ $this->restrictToPeriods = array_intersect($this->restrictToPeriods, PeriodFactory::getPeriodsEnabledForAPI());
1674	+
1675	+ return $this->restrictToPeriods;
1676	+ }
1677	+
1678	+ /**
1679	+ * @return array
1680	+ */
1681	+ private function getDefaultPeriodsToProcess()
1682	+ {
1683	+ return array('day', 'week', 'month', 'year', 'range');
1684	+ }
1685	+
1686	+ /**
1687	+ * @param $idSite
1688	+ * @return bool
1689	+ */
1690	+ private function isOldReportInvalidatedForWebsite($idSite)
1691	+ {
1692	+ return in_array($idSite, $this->idSitesInvalidatedOldReports);
1693	+ }
1694	+
1695	+ private function isWebsiteUsingTheTracker($idSite)
1696	+ {
1697	+ if (!isset($this->idSitesNotUsingTracker)) {
1698	+ // we want to trigger event only once
1699	+ $this->idSitesNotUsingTracker = array();
1700	+
1701	+ /**
1702	+ * This event is triggered when detecting whether there are sites that do not use the tracker.
1703	+ *
1704	+ * By default we only archive a site when there was actually any visit since the last archiving.
1705	+ * However, some plugins do import data from another source instead of using the tracker and therefore
1706	+ * will never have any visits for this site. To make sure we still archive data for such a site when
1707	+ * archiving for this site is requested, you can listen to this event and add the idSite to the list of
1708	+ * sites that do not use the tracker.
1709	+ *
1710	+ * @param bool $idSitesNotUsingTracker The list of idSites that rather import data instead of using the tracker
1711	+ */
1712	+ Piwik::postEvent('CronArchive.getIdSitesNotUsingTracker', array(&$this->idSitesNotUsingTracker));
1713	+
1714	+ if (!empty($this->idSitesNotUsingTracker)) {
1715	+ $this->logger->info("- The following websites do not use the tracker: " . implode(',', $this->idSitesNotUsingTracker));
1716	+ }
1717	+ }
1718	+
1719	+ $isUsingTracker = !in_array($idSite, $this->idSitesNotUsingTracker);
1720	+
1721	+ return $isUsingTracker;
1722	+ }
1723	+
1724	+ private function shouldProcessPeriod($period)
1725	+ {
1726	+ if (empty($this->shouldArchiveOnlySpecificPeriods)) {
1727	+ return true;
1728	+ }
1729	+
1730	+ return in_array($period, $this->shouldArchiveOnlySpecificPeriods);
1731	+ }
1732	+
1733	+ /**
1734	+ * @param $idSite
1735	+ * @param $period
1736	+ * @param $lastTimestampWebsiteProcessed
1737	+ * @return string
1738	+ */
1739	+ private function getDateLastN($idSite, $period, $lastTimestampWebsiteProcessed)
1740	+ {
1741	+ $dateLastMax = self::DEFAULT_DATE_LAST;
1742	+ if ($period == 'year') {
1743	+ $dateLastMax = self::DEFAULT_DATE_LAST_YEARS;
1744	+ } elseif ($period == 'week') {
1745	+ $dateLastMax = self::DEFAULT_DATE_LAST_WEEKS;
1746	+ }
1747	+ if (empty($lastTimestampWebsiteProcessed)) {
1748	+ $creationDateFor = \Piwik\Site::getCreationDateFor($idSite);
1749	+ $lastTimestampWebsiteProcessed = strtotime($creationDateFor);
1750	+ }
1751	+
1752	+ // Enforcing last2 at minimum to work around timing issues and ensure we make most archives available
1753	+ $dateLast = floor((time() - $lastTimestampWebsiteProcessed) / 86400) + 2;
1754	+ if ($dateLast > $dateLastMax) {
1755	+ $dateLast = $dateLastMax;
1756	+ }
1757	+
1758	+ if (!empty($this->dateLastForced)) {
1759	+ $dateLast = $this->dateLastForced;
1760	+ }
1761	+
1762	+ return "last" . $dateLast;
1763	+ }
1764	+
1765	+ /**
1766	+ * @return int
1767	+ */
1768	+ private function getConcurrentRequestsPerWebsite()
1769	+ {
1770	+ if (false !== $this->concurrentRequestsPerWebsite) {
1771	+ return $this->concurrentRequestsPerWebsite;
1772	+ }
1773	+
1774	+ return self::MAX_CONCURRENT_API_REQUESTS;
1775	+ }
1776	+
1777	+ /**
1778	+ * @param $idSite
1779	+ * @return false\|string
1780	+ */
1781	+ private function getPeriodLastProcessedTimestamp($idSite)
1782	+ {
1783	+ $timestamp = Option::get($this->lastRunKey($idSite, "periods"));
1784	+ return $this->sanitiseTimestamp($timestamp);
1785	+ }
1786	+
1787	+ /**
1788	+ * @param $idSite
1789	+ * @return false\|string
1790	+ */
1791	+ private function getDayLastProcessedTimestamp($idSite)
1792	+ {
1793	+ $timestamp = Option::get($this->lastRunKey($idSite, "day"));
1794	+ return $this->sanitiseTimestamp($timestamp);
1795	+ }
1796	+
1797	+ /**
1798	+ * @return false\|string
1799	+ */
1800	+ private function getLastSuccessRunTimestamp()
1801	+ {
1802	+ $timestamp = Option::get(self::OPTION_ARCHIVING_FINISHED_TS);
1803	+ return $this->sanitiseTimestamp($timestamp);
1804	+ }
1805	+
1806	+ private function sanitiseTimestamp($timestamp)
1807	+ {
1808	+ $now = time();
1809	+ return ($timestamp < $now) ? $timestamp : $now;
1810	+ }
1811	+
1812	+ /**
1813	+ * @param $idSite
1814	+ * @return array of date strings
1815	+ */
1816	+ private function getCustomDateRangeToPreProcess($idSite)
1817	+ {
1818	+ static $cache = null;
1819	+ if (is_null($cache)) {
1820	+ $cache = $this->loadCustomDateRangeToPreProcess();
1821	+ }
1822	+
1823	+ if (empty($cache[$idSite])) {
1824	+ $cache[$idSite] = array();
1825	+ }
1826	+
1827	+ $customRanges = array_filter(Config::getInstance()->General['archiving_custom_ranges']);
1828	+
1829	+ if (!empty($customRanges)) {
1830	+ $cache[$idSite] = array_merge($cache[$idSite], $customRanges);
1831	+ }
1832	+
1833	+ $dates = array_unique($cache[$idSite]);
1834	+ return $dates;
1835	+ }
1836	+
1837	+ /**
1838	+ * @return array
1839	+ */
1840	+ private function loadCustomDateRangeToPreProcess()
1841	+ {
1842	+ $customDateRangesToProcessForSites = array();
1843	+
1844	+ // For all users who have selected this website to load by default,
1845	+ // we load the default period/date that will be loaded for this user
1846	+ // and make sure it's pre-archived
1847	+ $allUsersPreferences = APIUsersManager::getInstance()->getAllUsersPreferences(array(
1848	+ APIUsersManager::PREFERENCE_DEFAULT_REPORT_DATE,
1849	+ APIUsersManager::PREFERENCE_DEFAULT_REPORT
1850	+ ));
1851	+
1852	+ foreach ($allUsersPreferences as $userLogin => $userPreferences) {
1853	+ if (!isset($userPreferences[APIUsersManager::PREFERENCE_DEFAULT_REPORT_DATE])) {
1854	+ continue;
1855	+ }
1856	+
1857	+ $defaultDate = $userPreferences[APIUsersManager::PREFERENCE_DEFAULT_REPORT_DATE];
1858	+ $preference = new UserPreferences();
1859	+ $period = $preference->getDefaultPeriod($defaultDate);
1860	+ if ($period != 'range') {
1861	+ continue;
1862	+ }
1863	+
1864	+ if (isset($userPreferences[APIUsersManager::PREFERENCE_DEFAULT_REPORT])
1865	+ && is_numeric($userPreferences[APIUsersManager::PREFERENCE_DEFAULT_REPORT])) {
1866	+ // If user selected one particular website ID
1867	+ $idSites = array($userPreferences[APIUsersManager::PREFERENCE_DEFAULT_REPORT]);
1868	+ } else {
1869	+ // If user selected "All websites" or some other random value, we pre-process all websites that they have access to
1870	+ $idSites = APISitesManager::getInstance()->getSitesIdWithAtLeastViewAccess($userLogin);
1871	+ }
1872	+
1873	+ foreach ($idSites as $idSite) {
1874	+ $customDateRangesToProcessForSites[$idSite][] = $defaultDate;
1875	+ }
1876	+ }
1877	+
1878	+ return $customDateRangesToProcessForSites;
1879	+ }
1880	+
1881	+ /**
1882	+ * @param $url
1883	+ * @return string
1884	+ */
1885	+ private function makeRequestUrl($url)
1886	+ {
1887	+ $url = $url . self::APPEND_TO_API_REQUEST;
1888	+
1889	+ if ($this->shouldStartProfiler) {
1890	+ $url .= "&xhprof=2";
1891	+ }
1892	+
1893	+ if ($this->testmode) {
1894	+ $url .= "&testmode=1";
1895	+ }
1896	+
1897	+ /**
1898	+ * @ignore
1899	+ */
1900	+ Piwik::postEvent('CronArchive.alterArchivingRequestUrl', [&$url]);
1901	+
1902	+ return $url;
1903	+ }
1904	+
1905	+ protected function wasSegmentChangedRecently($definition, $allSegments)
1906	+ {
1907	+ foreach ($allSegments as $segment) {
1908	+ if ($segment['definition'] === $definition) {
1909	+ $twentyFourHoursAgo = Date::now()->subHour(24);
1910	+ $segmentDate = $segment['ts_created'];
1911	+ if (!empty($segment['ts_last_edit'])) {
1912	+ $segmentDate = $segment['ts_last_edit'];
1913	+ }
1914	+ return Date::factory($segmentDate)->isLater($twentyFourHoursAgo);
1915	+ }
1916	+ }
1917	+
1918	+ return false;
1919	+ }
1920	+
1921	+ /**
1922	+ * @param $idSite
1923	+ * @param $period
1924	+ * @param $date
1925	+ * @return Request[]
1926	+ */
1927	+ private function getUrlsWithSegment($idSite, $period, $date)
1928	+ {
1929	+ $urlsWithSegment = array();
1930	+ $segmentsForSite = $this->getSegmentsForSite($idSite);
1931	+
1932	+ $segments = array();
1933	+ foreach ($segmentsForSite as $segment) {
1934	+ if ($this->shouldSkipSegmentArchiving($segment)) {
1935	+ $this->logger->info("- skipping segment archiving for '{segment}'.", array('segment' => $segment));
1936	+
1937	+ continue;
1938	+ }
1939	+
1940	+ $segments[] = $segment;
1941	+ }
1942	+
1943	+
1944	+ $segmentCount = count($segments);
1945	+ $processedSegmentCount = 0;
1946	+
1947	+ $allSegmentsFullInfo = array();
1948	+ if ($this->skipSegmentsToday) {
1949	+ // small performance tweak... only needed when skip segments today
1950	+ $segmentEditorModel = StaticContainer::get('Piwik\Plugins\SegmentEditor\Model');
1951	+ $allSegmentsFullInfo = $segmentEditorModel->getSegmentsToAutoArchive($idSite);
1952	+ }
1953	+
1954	+ foreach ($segments as $segment) {
1955	+ $shouldSkipToday = $this->skipSegmentsToday && !$this->wasSegmentChangedRecently($segment, $allSegmentsFullInfo);
1956	+
1957	+ if ($this->skipSegmentsToday && !$shouldSkipToday) {
1958	+ $this->logger->info(sprintf('Segment "%s" was created or changed recently and will therefore archive today', $segment));
1959	+ }
1960	+
1961	+ $dateParamForSegment = $this->segmentArchivingRequestUrlProvider->getUrlParameterDateString($idSite, $period, $date, $segment);
1962	+
1963	+ $urlWithSegment = $this->getVisitsRequestUrl($idSite, $period, $dateParamForSegment, $segment);
1964	+ $urlWithSegment = $this->makeRequestUrl($urlWithSegment);
1965	+
1966	+ if ($shouldSkipToday) {
1967	+ $urlWithSegment .= '&skipArchiveSegmentToday=1';
1968	+ }
1969	+
1970	+ if ($this->isAlreadyArchivingSegment($urlWithSegment, $idSite, $period, $segment)) {
1971	+ continue;
1972	+ }
1973	+
1974	+ $request = new Request($urlWithSegment);
1975	+ $logger = $this->logger;
1976	+ $self = $this;
1977	+ $request->before(function () use ($logger, $segment, $segmentCount, &$processedSegmentCount, $idSite, $period, $date, $urlWithSegment, $self, $request) {
1978	+ if ($self->isAlreadyArchivingSegment($urlWithSegment, $idSite, $period, $segment)) {
1979	+ return Request::ABORT;
1980	+ }
1981	+
1982	+ list($isThereArchive, $newDate) = $this->isThereAValidArchiveForPeriod($idSite, $period, $date, $segment);
1983	+ if ($isThereArchive) {
1984	+ $this->logArchiveWebsiteSkippedValidArchiveExists($idSite, $period, $date, $segment);
1985	+ return Request::ABORT;
1986	+ }
1987	+
1988	+ $urlBefore = $request->getUrl();
1989	+ $url = preg_replace('/([&?])date=[^&]*/', '$1date=' . $newDate, $urlBefore);
1990	+ $request->setUrl($url);
1991	+ $request->makeSureDateIsNotSingleDayRange();
1992	+
1993	+ // check again if we are already archiving the URL since we just changed it
1994	+ if ($request->getUrl() !== $urlBefore
1995	+ && $self->isAlreadyArchivingSegment($request->getUrl(), $idSite, $period, $segment)
1996	+ ) {
1997	+ return Request::ABORT;
1998	+ }
1999	+
2000	+ $processedSegmentCount++;
2001	+ $logger->info(sprintf(
2002	+ '- pre-processing segment %d/%d %s [date = %s]',
2003	+ $processedSegmentCount,
2004	+ $segmentCount,
2005	+ $segment,
2006	+ $newDate
2007	+ ));
2008	+ });
2009	+
2010	+ $urlsWithSegment[] = $request;
2011	+ }
2012	+
2013	+ return $urlsWithSegment;
2014	+ }
2015	+
2016	+ private function isAlreadyArchivingAnyLowerOrThisPeriod($idSite, $period, $segment = false)
2017	+ {
2018	+ $periodOrder = array('day', 'week', 'month', 'year');
2019	+ $cliMulti = $this->makeCliMulti();
2020	+
2021	+ $index = array_search($period, $periodOrder);
2022	+ if ($index !== false) {
2023	+ // we only need to check for week, month, year if any earlier period is already running
2024	+ // so when period = month, then we check for day and week
2025	+
2026	+ for ($i = 0; $i <= $index; $i++) {
2027	+ $periodToCheck = $periodOrder[$i];
2028	+
2029	+ // the date will be ignored in isCommandAlreadyRunning() because it could be any date
2030	+ $urlCheck = $this->getVisitsRequestUrl($idSite, $periodToCheck, 'last2', $segment);
2031	+ $urlCheck = $this->makeRequestUrl($urlCheck);
2032	+
2033	+ if ($cliMulti->isCommandAlreadyRunning($urlCheck)) {
2034	+ return $periodToCheck;
2035	+ }
2036	+ }
2037	+ }
2038	+
2039	+ return false;
2040	+ }
2041	+
2042	+ private function createSitesToArchiveQueue($websitesIds)
2043	+ {
2044	+ // use synchronous, single process queue if --force-idsites is used or sharing site IDs isn't supported
2045	+ if (!SharedSiteIds::isSupported() \|\| !empty($this->shouldArchiveSpecifiedSites)) {
2046	+ return new FixedSiteIds($websitesIds);
2047	+ }
2048	+
2049	+ // use separate shared queue if --force-all-websites is used
2050	+ if (!empty($this->shouldArchiveAllSites)) {
2051	+ return new SharedSiteIds($websitesIds, SharedSiteIds::OPTION_ALL_WEBSITES);
2052	+ }
2053	+
2054	+ return new SharedSiteIds($websitesIds);
2055	+ }
2056	+
2057	+ /**
2058	+ * @param $idSite
2059	+ * @param $period
2060	+ * @param $date
2061	+ */
2062	+ private function logArchiveWebsite($idSite, $period, $date)
2063	+ {
2064	+ $this->logger->info(sprintf(
2065	+ "Will pre-process for website id = %s, period = %s, date = %s",
2066	+ $idSite,
2067	+ $period,
2068	+ $date
2069	+ ));
2070	+ $this->logger->info('- pre-processing all visits');
2071	+ }
2072	+
2073	+ public function isAlreadyArchivingUrl($url, $idSite, $period, $date)
2074	+ {
2075	+ $periodInProgress = $this->isAlreadyArchivingAnyLowerOrThisPeriod($idSite, $period);
2076	+ if ($periodInProgress) {
2077	+ $this->logger->info("- skipping archiving for period '{period}' because processing the period '{periodcheck}' is already in progress.", array('period' => $period, 'periodcheck' => $periodInProgress));
2078	+ return true;
2079	+ }
2080	+
2081	+ $cliMulti = $this->makeCliMulti();
2082	+ if ($cliMulti->isCommandAlreadyRunning($url)) {
2083	+ $this->logArchiveWebsiteAlreadyInProcess($idSite, $period, $date);
2084	+ return true;
2085	+ }
2086	+ return false;
2087	+ }
2088	+
2089	+ public function isAlreadyArchivingSegment($urlWithSegment, $idSite, $period, $segment)
2090	+ {
2091	+ // we can check for this or lower period only when the below condition is given. Otherwise the archiver might launch
2092	+ // the following requests at once:
2093	+ // - week
2094	+ // - week segment1
2095	+ // - week segment2
2096	+ // and it would always skip archiving the segments cause the week was launched first and would be running when
2097	+ // it starts them all 3 "at the same time".
2098	+ $isProcessingOne = $this->concurrentRequestsPerWebsite == 1;
2099	+
2100	+ $periodInProgress = $isProcessingOne && $this->isAlreadyArchivingAnyLowerOrThisPeriod($idSite, $period);
2101	+ if ($periodInProgress) {
2102	+ $this->logger->info("- skipping segment archiving for period '{period}' with segment '{segment}' because processing the period '{periodcheck}' is already in progress.", array('segment' => $segment, 'period' => $period, 'periodcheck' => $periodInProgress));
2103	+ return true;
2104	+ }
2105	+
2106	+ $cliMulti = $this->makeCliMulti();
2107	+ if ($cliMulti->isCommandAlreadyRunning($urlWithSegment)) {
2108	+ $this->logger->info("- skipping segment archiving for '{segment}' because such a process is already in progress.", array('segment' => $segment));
2109	+ return true;
2110	+ }
2111	+
2112	+ return false;
2113	+ }
2114	+
2115	+ /**
2116	+ * @param $idSite
2117	+ * @param $period
2118	+ * @param $date
2119	+ */
2120	+ private function logArchiveWebsiteAlreadyInProcess($idSite, $period, $date)
2121	+ {
2122	+ $this->logger->info(sprintf(
2123	+ "Will not pre-process for website id = %s, period = %s, date = %s because such a process is already in progress.",
2124	+ $idSite,
2125	+ $period,
2126	+ $date
2127	+ ));
2128	+ }
2129	+
2130	+ private function shouldSkipSegmentArchiving($segment)
2131	+ {
2132	+ if ($this->disableSegmentsArchiving) {
2133	+ return true;
2134	+ }
2135	+
2136	+ return !empty($this->segmentsToForce) && !in_array($segment, $this->segmentsToForce);
2137	+ }
2138	+
2139	+ private function logForcedSegmentInfo()
2140	+ {
2141	+ if (empty($this->segmentsToForce)) {
2142	+ return;
2143	+ }
2144	+
2145	+ $this->logger->info("- Limiting segment archiving to following segments:");
2146	+ foreach ($this->segmentsToForce as $segmentDefinition) {
2147	+ $this->logger->info(" * " . $segmentDefinition);
2148	+ }
2149	+ }
2150	+
2151	+ /**
2152	+ * @return CliMulti
2153	+ */
2154	+ private function makeCliMulti()
2155	+ {
2156	+ /** @var CliMulti $cliMulti */
2157	+ $cliMulti = StaticContainer::getContainer()->make('Piwik\CliMulti');
2158	+ $cliMulti->setUrlToPiwik($this->urlToPiwik);
2159	+ $cliMulti->setPhpCliConfigurationOptions($this->phpCliConfigurationOptions);
2160	+ $cliMulti->setAcceptInvalidSSLCertificate($this->acceptInvalidSSLCertificate);
2161	+ $cliMulti->setConcurrentProcessesLimit($this->getConcurrentRequestsPerWebsite());
2162	+ $cliMulti->runAsSuperUser();
2163	+ $cliMulti->onProcessFinish(function ($pid) {
2164	+ $this->printPerformanceStatsForProcess($pid);
2165	+ });
2166	+ return $cliMulti;
2167	+ }
2168	+
2169	+ public function setUrlToPiwik($url)
2170	+ {
2171	+ $this->urlToPiwik = $url;
2172	+ }
2173	+
2174	+ private function printPerformanceStatsForProcess($childPid)
2175	+ {
2176	+ if (!$this->isArchiveProfilingEnabled) {
2177	+ return;
2178	+ }
2179	+
2180	+ $data = Logger::getMeasurementsFor(getmypid(), $childPid);
2181	+ if (empty($data)) {
2182	+ return;
2183	+ }
2184	+
2185	+ $message = "";
2186	+ foreach ($data as $request => $measurements) {
2187	+ $message .= "PERFORMANCE FOR " . $request . "\n ";
2188	+ $message .= implode("\n ", $measurements) . "\n";
2189	+ }
2190	+ $this->logger->info($message);
2191	+ }
2192	+ }

app/core/CronArchive/FixedSiteIds.php ADDED Viewed

	@@ -0,0 +1,65 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik\CronArchive;
10	+
11	+
12	+ class FixedSiteIds
13	+ {
14	+ private $siteIds = array();
15	+ private $index = -1;
16	+
17	+ public function __construct($websiteIds)
18	+ {
19	+ if (!empty($websiteIds)) {
20	+ $this->siteIds = array_values($websiteIds);
21	+ }
22	+ }
23	+
24	+ public function getInitialSiteIds()
25	+ {
26	+ return $this->siteIds;
27	+ }
28	+
29	+ /**
30	+ * Get the number of total websites that needs to be processed.
31	+ *
32	+ * @return int
33	+ */
34	+ public function getNumSites()
35	+ {
36	+ return count($this->siteIds);
37	+ }
38	+
39	+ /**
40	+ * Get the number of already processed websites. All websites were processed by the current archiver.
41	+ *
42	+ * @return int
43	+ */
44	+ public function getNumProcessedWebsites()
45	+ {
46	+ $numProcessed = $this->index + 1;
47	+
48	+ if ($numProcessed > $this->getNumSites()) {
49	+ return $this->getNumSites();
50	+ }
51	+
52	+ return $numProcessed;
53	+ }
54	+
55	+ public function getNextSiteId()
56	+ {
57	+ $this->index++;
58	+
59	+ if (!empty($this->siteIds[$this->index])) {
60	+ return $this->siteIds[$this->index];
61	+ }
62	+
63	+ return null;
64	+ }
65	+ }

app/core/CronArchive/Performance/Logger.php ADDED Viewed

	@@ -0,0 +1,119 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ */
8	+
9	+ namespace Piwik\CronArchive\Performance;
10	+
11	+ use Piwik\ArchiveProcessor;
12	+ use Piwik\Common;
13	+ use Piwik\Config;
14	+ use Piwik\Option;
15	+ use Piwik\Timer;
16	+ use Piwik\Url;
17	+ use Psr\Log\LoggerInterface;
18	+
19	+ class Logger
20	+ {
21	+ /**
22	+ * @var int
23	+ */
24	+ private $isEnabled;
25	+
26	+ /**
27	+ * @var LoggerInterface
28	+ */
29	+ private $logger;
30	+
31	+ /**
32	+ * @var int
33	+ */
34	+ private $archivingRunId;
35	+
36	+ public function __construct(Config $config, LoggerInterface $logger = null)
37	+ {
38	+ $this->isEnabled = $config->Debug['archiving_profile'] == 1;
39	+ $this->logger = $logger;
40	+
41	+ $this->archivingRunId = $this->getArchivingRunId();
42	+ if (empty($this->archivingRunId)) {
43	+ $this->isEnabled = false;
44	+ }
45	+ }
46	+
47	+ public function logMeasurement($category, $name, ArchiveProcessor\Parameters $activeArchivingParams, Timer $timer)
48	+ {
49	+ if (!$this->isEnabled \|\| !$this->logger) {
50	+ return;
51	+ }
52	+
53	+ $measurement = new Measurement($category, $name, $activeArchivingParams->getSite()->getId(),
54	+ $activeArchivingParams->getPeriod()->getRangeString(), $activeArchivingParams->getPeriod()->getLabel(),
55	+ $activeArchivingParams->getSegment()->getString(), $timer->getTime(), $timer->getMemoryLeakValue(),
56	+ $timer->getPeakMemoryValue());
57	+
58	+ $params = array_merge($_GET);
59	+ unset($params['pid']);
60	+ unset($params['runid']);
61	+
62	+ $this->logger->info("[runid={runid},pid={pid}] {request}: {measurement}", [
63	+ 'pid' => Common::getRequestVar('pid', false),
64	+ 'runid' => $this->getArchivingRunId(),
65	+ 'request' => Url::getQueryStringFromParameters($params),
66	+ 'measurement' => $measurement,
67	+ ]);
68	+ }
69	+
70	+ public static function getMeasurementsFor($runId, $childPid)
71	+ {
72	+ $profilingLogFile = preg_replace('/[\'"]/', '', Config::getInstance()->Debug['archive_profiling_log']);
73	+ if (!is_readable($profilingLogFile)) {
74	+ return [];
75	+ }
76	+
77	+ $runId = self::cleanId($runId);
78	+ $childPid = self::cleanId($childPid);
79	+
80	+ $lineIdentifier = "[runid=$runId,pid=$childPid]";
81	+ $lines = `grep "$childPid" "$profilingLogFile"`;
82	+ $lines = explode("\n", $lines);
83	+ $lines = array_map(function ($line) use ($lineIdentifier) {
84	+ $index = strpos($line, $lineIdentifier);
85	+ if ($index === false) {
86	+ return null;
87	+ }
88	+ $line = substr($line, $index + strlen($lineIdentifier));
89	+ return trim($line);
90	+ }, $lines);
91	+ $lines = array_filter($lines);
92	+ $lines = array_map(function ($line) {
93	+ $parts = explode(":", $line, 2);
94	+ $parts = array_map('trim', $parts);
95	+ return $parts;
96	+ }, $lines);
97	+
98	+ $data = [];
99	+ foreach ($lines as $line) {
100	+ if (count($line) != 2) {
101	+ continue;
102	+ }
103	+
104	+ list($request, $measurement) = $line;
105	+ $data[$request][] = $measurement;
106	+ }
107	+ return $data;
108	+ }
109	+
110	+ private function getArchivingRunId()
111	+ {
112	+ return Common::getRequestVar('runid', false);
113	+ }
114	+
115	+ private static function cleanId($id)
116	+ {
117	+ return preg_replace('/[^a-zA-Z0-9_-]/', '', $id);
118	+ }
119	+ }

app/core/CronArchive/Performance/Measurement.php ADDED Viewed

	@@ -0,0 +1,165 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ */
8	+
9	+ namespace Piwik\CronArchive\Performance;
10	+
11	+ class Measurement
12	+ {
13	+ /**
14	+ * @var string
15	+ */
16	+ private $category;
17	+
18	+ /**
19	+ * @var string
20	+ */
21	+ private $measuredName;
22	+
23	+ /**
24	+ * @var string
25	+ */
26	+ private $idSite;
27	+
28	+ /**
29	+ * @var string
30	+ */
31	+ private $dateRange;
32	+
33	+ /**
34	+ * @var string
35	+ */
36	+ private $periodType;
37	+
38	+ /**
39	+ * @var string
40	+ */
41	+ private $segment;
42	+
43	+ /**
44	+ * @var float
45	+ */
46	+ private $time;
47	+
48	+ /**
49	+ * @var string
50	+ */
51	+ private $memory;
52	+
53	+ /**
54	+ * @var string
55	+ */
56	+ private $peakMemory;
57	+
58	+ public function __construct($category, $name, $idSite, $dateRange, $periodType, $segment, $time, $memory, $peakMemory)
59	+ {
60	+ $this->category = $category;
61	+ $this->measuredName = $name;
62	+ $this->idSite = $idSite;
63	+ $this->dateRange = $dateRange;
64	+ $this->periodType = $periodType;
65	+ $this->segment = trim($segment);
66	+ $this->time = $time;
67	+ $this->memory = $memory;
68	+ $this->peakMemory = $peakMemory;
69	+ }
70	+
71	+ public function __toString()
72	+ {
73	+ $parts = [
74	+ ucfirst($this->category) . ": {$this->measuredName}",
75	+ "idSite: {$this->idSite}",
76	+ "period: {$this->periodType} ({$this->dateRange})",
77	+ "segment: " . (!empty($this->segment) ? $this->segment : 'none'),
78	+ "duration: {$this->time}s",
79	+ "memory leak: {$this->memory}",
80	+ "peak memory usage: {$this->peakMemory}",
81	+ ];
82	+
83	+ return implode(', ', $parts);
84	+ }
85	+
86	+ /**
87	+ * @return string
88	+ */
89	+ public function getCategory()
90	+ {
91	+ return $this->category;
92	+ }
93	+
94	+ /**
95	+ * @param string $category
96	+ */
97	+ public function setCategory($category)
98	+ {
99	+ $this->category = $category;
100	+ }
101	+
102	+ /**
103	+ * @return string
104	+ */
105	+ public function getMeasuredName()
106	+ {
107	+ return $this->measuredName;
108	+ }
109	+
110	+ /**
111	+ * @param string $measuredName
112	+ */
113	+ public function setMeasuredName($measuredName)
114	+ {
115	+ $this->measuredName = $measuredName;
116	+ }
117	+
118	+ /**
119	+ * @return string
120	+ */
121	+ public function getIdSite()
122	+ {
123	+ return $this->idSite;
124	+ }
125	+
126	+ /**
127	+ * @param string $idSite
128	+ */
129	+ public function setIdSite($idSite)
130	+ {
131	+ $this->idSite = $idSite;
132	+ }
133	+
134	+ /**
135	+ * @return string
136	+ */
137	+ public function getDateRange()
138	+ {
139	+ return $this->dateRange;
140	+ }
141	+
142	+ /**
143	+ * @param string $dateRange
144	+ */
145	+ public function setDateRange($dateRange)
146	+ {
147	+ $this->dateRange = $dateRange;
148	+ }
149	+
150	+ /**
151	+ * @return string
152	+ */
153	+ public function getPeriodType()
154	+ {
155	+ return $this->periodType;
156	+ }
157	+
158	+ /**
159	+ * @param string $periodType
160	+ */
161	+ public function setPeriodType($periodType)
162	+ {
163	+ $this->periodType = $periodType;
164	+ }
165	+ }

app/core/CronArchive/SegmentArchivingRequestUrlProvider.php ADDED Viewed

	@@ -0,0 +1,208 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ */
8	+ namespace Piwik\CronArchive;
9	+
10	+ use Piwik\Cache\Cache;
11	+ use Piwik\Cache\Transient;
12	+ use Piwik\Container\StaticContainer;
13	+ use Piwik\Date;
14	+ use Piwik\Period\Factory as PeriodFactory;
15	+ use Piwik\Period\Range;
16	+ use Piwik\Plugins\SegmentEditor\Model;
17	+ use Psr\Log\LoggerInterface;
18	+
19	+ /**
20	+ * Provides URLs that initiate archiving during cron archiving for segments.
21	+ *
22	+ * Handles the `[General] process_new_segments_from` INI option.
23	+ */
24	+ class SegmentArchivingRequestUrlProvider
25	+ {
26	+ const BEGINNING_OF_TIME = 'beginning_of_time';
27	+ const CREATION_TIME = 'segment_creation_time';
28	+ const LAST_EDIT_TIME = 'segment_last_edit_time';
29	+
30	+ /**
31	+ * @var Model
32	+ */
33	+ private $segmentEditorModel;
34	+
35	+ /**
36	+ * @var Cache
37	+ */
38	+ private $segmentListCache;
39	+
40	+ /**
41	+ * @var Date
42	+ */
43	+ private $now;
44	+
45	+ private $processNewSegmentsFrom;
46	+
47	+ /**
48	+ * @var LoggerInterface
49	+ */
50	+ private $logger;
51	+
52	+ public function __construct($processNewSegmentsFrom, Model $segmentEditorModel = null, Cache $segmentListCache = null,
53	+ Date $now = null, LoggerInterface $logger = null)
54	+ {
55	+ $this->processNewSegmentsFrom = $processNewSegmentsFrom;
56	+ $this->segmentEditorModel = $segmentEditorModel ?: new Model();
57	+ $this->segmentListCache = $segmentListCache ?: new Transient();
58	+ $this->now = $now ?: Date::factory('now');
59	+ $this->logger = $logger ?: StaticContainer::get('Psr\Log\LoggerInterface');
60	+ }
61	+
62	+ public function getUrlParameterDateString($idSite, $period, $date, $segment)
63	+ {
64	+ $oldestDateToProcessForNewSegment = $this->getOldestDateToProcessForNewSegment($idSite, $segment);
65	+ if (empty($oldestDateToProcessForNewSegment)) {
66	+ return $date;
67	+ }
68	+
69	+ // if the start date for the archiving request is before the minimum date allowed for processing this segment,
70	+ // use the minimum allowed date as the start date
71	+ $periodObj = PeriodFactory::build($period, $date);
72	+ if ($periodObj->getDateStart()->getTimestamp() < $oldestDateToProcessForNewSegment->getTimestamp()) {
73	+ $this->logger->debug("Start date of archiving request period ({start}) is older than configured oldest date to process for the segment.", array(
74	+ 'start' => $periodObj->getDateStart()
75	+ ));
76	+
77	+ $endDate = $periodObj->getDateEnd();
78	+
79	+ // if the creation time of a segment is older than the end date of the archiving request range, we cannot
80	+ // blindly rewrite the date string, since the resulting range would be incorrect. instead we make the
81	+ // start date equal to the end date, so less archiving occurs, and no fatal error occurs.
82	+ if ($oldestDateToProcessForNewSegment->getTimestamp() > $endDate->getTimestamp()) {
83	+ $this->logger->debug("Oldest date to process is greater than end date of archiving request period ({end}), so setting oldest date to end date.", array(
84	+ 'end' => $endDate
85	+ ));
86	+
87	+ $oldestDateToProcessForNewSegment = $endDate;
88	+ }
89	+
90	+ $date = $oldestDateToProcessForNewSegment->toString().','.$endDate;
91	+
92	+ $this->logger->debug("Archiving request date range changed to {date} w/ period {period}.", array('date' => $date, 'period' => $period));
93	+ }
94	+
95	+ return $date;
96	+ }
97	+
98	+ private function getOldestDateToProcessForNewSegment($idSite, $segment)
99	+ {
100	+ /**
101	+ * @var Date $segmentCreatedTime
102	+ * @var Date $segmentLastEditedTime
103	+ */
104	+ list($segmentCreatedTime, $segmentLastEditedTime) = $this->getCreatedTimeOfSegment($idSite, $segment);
105	+
106	+ if ($this->processNewSegmentsFrom == self::CREATION_TIME) {
107	+ $this->logger->debug("process_new_segments_from set to segment_creation_time, oldest date to process is {time}", array('time' => $segmentCreatedTime));
108	+
109	+ return $segmentCreatedTime;
110	+ } elseif ($this->processNewSegmentsFrom == self::LAST_EDIT_TIME) {
111	+ $this->logger->debug("process_new_segments_from set to segment_last_edit_time, segment last edit time is {time}",
112	+ array('time' => $segmentLastEditedTime));
113	+
114	+ if ($segmentLastEditedTime === null
115	+ \|\| $segmentLastEditedTime->getTimestamp() < $segmentCreatedTime->getTimestamp()
116	+ ) {
117	+ $this->logger->debug("segment last edit time is older than created time, using created time instead");
118	+
119	+ $segmentLastEditedTime = $segmentCreatedTime;
120	+ }
121	+
122	+ return $segmentLastEditedTime;
123	+ } elseif (preg_match("/^last([0-9]+)$/", $this->processNewSegmentsFrom, $matches)) {
124	+ $lastN = $matches[1];
125	+
126	+ list($lastDate, $lastPeriod) = Range::getDateXPeriodsAgo($lastN, $segmentCreatedTime, 'day');
127	+ $result = Date::factory($lastDate);
128	+
129	+ $this->logger->debug("process_new_segments_from set to last{N}, oldest date to process is {time}", array('N' => $lastN, 'time' => $result));
130	+
131	+ return $result;
132	+ } else {
133	+ $this->logger->debug("process_new_segments_from set to beginning_of_time or cannot recognize value");
134	+
135	+ return null;
136	+ }
137	+ }
138	+
139	+ private function getCreatedTimeOfSegment($idSite, $segmentDefinition)
140	+ {
141	+ $segments = $this->getAllSegments();
142	+
143	+ /** @var Date $latestEditTime */
144	+ $latestEditTime = null;
145	+ $earliestCreatedTime = $this->now;
146	+ foreach ($segments as $segment) {
147	+ if (empty($segment['ts_created'])
148	+ \|\| empty($segment['definition'])
149	+ \|\| !isset($segment['enable_only_idsite'])
150	+ ) {
151	+ continue;
152	+ }
153	+
154	+ if ($this->isSegmentForSite($segment, $idSite)
155	+ && $segment['definition'] == $segmentDefinition
156	+ ) {
157	+ // check for an earlier ts_created timestamp
158	+ $createdTime = Date::factory($segment['ts_created']);
159	+ if ($createdTime->getTimestamp() < $earliestCreatedTime->getTimestamp()) {
160	+ $earliestCreatedTime = $createdTime;
161	+ }
162	+
163	+ // if there is no ts_last_edit timestamp, initialize it to ts_created
164	+ if (empty($segment['ts_last_edit'])) {
165	+ $segment['ts_last_edit'] = $segment['ts_created'];
166	+ }
167	+
168	+ // check for a later ts_last_edit timestamp
169	+ $lastEditTime = Date::factory($segment['ts_last_edit']);
170	+ if ($latestEditTime === null
171	+ \|\| $latestEditTime->getTimestamp() < $lastEditTime->getTimestamp()
172	+ ) {
173	+ $latestEditTime = $lastEditTime;
174	+ }
175	+ }
176	+ }
177	+
178	+ $this->logger->debug(
179	+ "Earliest created time of segment '{segment}' w/ idSite = {idSite} is found to be {createdTime}. Latest " .
180	+ "edit time is found to be {latestEditTime}.",
181	+ array(
182	+ 'segment' => $segmentDefinition,
183	+ 'idSite' => $idSite,
184	+ 'createdTime' => $earliestCreatedTime,
185	+ 'latestEditTime' => $latestEditTime,
186	+ )
187	+ );
188	+
189	+ return array($earliestCreatedTime, $latestEditTime);
190	+ }
191	+
192	+ private function getAllSegments()
193	+ {
194	+ if (!$this->segmentListCache->contains('all')) {
195	+ $segments = $this->segmentEditorModel->getAllSegmentsAndIgnoreVisibility();
196	+
197	+ $this->segmentListCache->save('all', $segments);
198	+ }
199	+
200	+ return $this->segmentListCache->fetch('all');
201	+ }
202	+
203	+ private function isSegmentForSite($segment, $idSite)
204	+ {
205	+ return $segment['enable_only_idsite'] == 0
206	+ \|\| $segment['enable_only_idsite'] == $idSite;
207	+ }
208	+ }

app/core/CronArchive/SharedSiteIds.php ADDED Viewed

	@@ -0,0 +1,202 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik\CronArchive;
10	+
11	+ use Exception;
12	+ use Piwik\CliMulti\Process;
13	+ use Piwik\Option;
14	+
15	+ /**
16	+ * This class saves all to be processed siteIds in an Option named 'SharedSiteIdsToArchive' and processes all sites
17	+ * within that list. If a user starts multiple archiver those archiver will help to finish processing that list.
18	+ */
19	+ class SharedSiteIds
20	+ {
21	+ const OPTION_DEFAULT = 'SharedSiteIdsToArchive';
22	+ const OPTION_ALL_WEBSITES = 'SharedSiteIdsToArchive_AllWebsites';
23	+
24	+ /**
25	+ * @var string
26	+ */
27	+ private $optionName;
28	+
29	+ private $siteIds = array();
30	+ private $currentSiteId;
31	+ private $done = false;
32	+ private $numWebsitesLeftToProcess;
33	+
34	+ public function __construct($websiteIds, $optionName = self::OPTION_DEFAULT)
35	+ {
36	+ $this->optionName = $optionName;
37	+
38	+ if (empty($websiteIds)) {
39	+ $websiteIds = array();
40	+ }
41	+
42	+ $self = $this;
43	+ $this->siteIds = $this->runExclusive(function () use ($self, $websiteIds) {
44	+ // if there are already sites to be archived registered, prefer the list of existing archive, meaning help
45	+ // to finish this queue of sites instead of starting a new queue
46	+ $existingWebsiteIds = $self->getAllSiteIdsToArchive();
47	+
48	+ if (!empty($existingWebsiteIds)) {
49	+ return $existingWebsiteIds;
50	+ }
51	+
52	+ $self->setSiteIdsToArchive($websiteIds);
53	+
54	+ return $websiteIds;
55	+ });
56	+ $this->numWebsitesLeftToProcess = $this->getNumSites();
57	+ }
58	+
59	+ public function getInitialSiteIds()
60	+ {
61	+ return $this->siteIds;
62	+ }
63	+
64	+ /**
65	+ * Get the number of total websites that needs to be processed.
66	+ *
67	+ * @return int
68	+ */
69	+ public function getNumSites()
70	+ {
71	+ return count($this->siteIds);
72	+ }
73	+
74	+ /**
75	+ * Get the number of already processed websites (not necessarily all of those where processed by this archiver).
76	+ *
77	+ * @return int
78	+ */
79	+ public function getNumProcessedWebsites()
80	+ {
81	+ if ($this->done) {
82	+ return $this->getNumSites();
83	+ }
84	+
85	+ if (empty($this->currentSiteId)) {
86	+ return 0;
87	+ }
88	+
89	+ $index = array_search($this->currentSiteId, $this->siteIds);
90	+
91	+ if (false === $index) {
92	+ return 0;
93	+ }
94	+
95	+ return $index + 1;
96	+ }
97	+
98	+ public function setSiteIdsToArchive($siteIds)
99	+ {
100	+ if (!empty($siteIds)) {
101	+ Option::set($this->optionName, implode(',', $siteIds));
102	+ } else {
103	+ Option::delete($this->optionName);
104	+ }
105	+ }
106	+
107	+ public function getAllSiteIdsToArchive()
108	+ {
109	+ Option::clearCachedOption($this->optionName);
110	+ $siteIdsToArchive = Option::get($this->optionName);
111	+
112	+ if (empty($siteIdsToArchive)) {
113	+ return array();
114	+ }
115	+
116	+ return explode(',', trim($siteIdsToArchive));
117	+ }
118	+
119	+ /**
120	+ * If there are multiple archiver running on the same node it makes sure only one of them performs an action and it
121	+ * will wait until another one has finished. Any closure you pass here should be very fast as other processes wait
122	+ * for this closure to finish otherwise. Currently only used for making multiple archivers at the same time work.
123	+ * If a closure takes more than 5 seconds we assume it is dead and simply continue.
124	+ *
125	+ * @param \Closure $closure
126	+ * @return mixed
127	+ * @throws \Exception
128	+ */
129	+ private function runExclusive($closure)
130	+ {
131	+ $process = new Process('archive.sharedsiteids');
132	+
133	+ while ($process->isRunning() && $process->getSecondsSinceCreation() < 5) {
134	+ // wait max 5 seconds, such an operation should not take longer
135	+ usleep(25 * 1000);
136	+ }
137	+
138	+ $process->startProcess();
139	+
140	+ try {
141	+ $result = $closure();
142	+ } catch (Exception $e) {
143	+ $process->finishProcess();
144	+ throw $e;
145	+ }
146	+
147	+ $process->finishProcess();
148	+
149	+ return $result;
150	+ }
151	+
152	+ /**
153	+ * Get the next site id that needs to be processed or null if all site ids where processed.
154	+ *
155	+ * @return int\|null
156	+ */
157	+ public function getNextSiteId()
158	+ {
159	+ if ($this->done) {
160	+ // we make sure we don't check again whether there are more sites to be archived as the list of
161	+ // sharedSiteIds may have been reset by now.
162	+ return null;
163	+ }
164	+
165	+ $self = $this;
166	+
167	+ $this->currentSiteId = $this->runExclusive(function () use ($self) {
168	+
169	+ $siteIds = $self->getAllSiteIdsToArchive();
170	+
171	+ if (empty($siteIds)) {
172	+ // done... no sites left to be archived
173	+ return null;
174	+ }
175	+
176	+ if (count($siteIds) > $self->numWebsitesLeftToProcess) {
177	+ // done... the number of siteIds in SharedSiteIds is larger than it was initially... therefore it must have
178	+ // been reset at some point.
179	+ return null;
180	+ }
181	+
182	+ $self->numWebsitesLeftToProcess = count($siteIds);
183	+
184	+ $nextSiteId = array_shift($siteIds);
185	+ $self->setSiteIdsToArchive($siteIds);
186	+
187	+ return $nextSiteId;
188	+ });
189	+
190	+ if (is_null($this->currentSiteId)) {
191	+ $this->done = true;
192	+ $this->numWebsitesLeftToProcess = 0;
193	+ }
194	+
195	+ return $this->currentSiteId;
196	+ }
197	+
198	+ public static function isSupported()
199	+ {
200	+ return Process::isSupported();
201	+ }
202	+ }

app/core/CronArchive/SitesToReprocessDistributedList.php ADDED Viewed

	@@ -0,0 +1,40 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik\CronArchive;
10	+
11	+ use Piwik\Concurrency\DistributedList;
12	+
13	+ /**
14	+ * Distributed list that stores the list of IDs of sites whose archives should be reprocessed.
15	+ *
16	+ * CronArchive will read this list of sites when archiving is being run, and make sure the sites
17	+ * are re-archived.
18	+ *
19	+ * Any class/API method/command/etc. is allowed to add site IDs to this list.
20	+ */
21	+ class SitesToReprocessDistributedList extends DistributedList
22	+ {
23	+ const OPTION_INVALIDATED_IDSITES_TO_REPROCESS = 'InvalidatedOldReports_WebsiteIds';
24	+
25	+ public function __construct()
26	+ {
27	+ parent::__construct(self::OPTION_INVALIDATED_IDSITES_TO_REPROCESS);
28	+ }
29	+
30	+ /**
31	+ * @inheritdoc
32	+ */
33	+ public function setAll($items)
34	+ {
35	+ $items = array_unique($items, SORT_REGULAR);
36	+ $items = array_values($items);
37	+
38	+ parent::setAll($items);
39	+ }
40	+ }

app/core/DataAccess/Actions.php ADDED Viewed

	@@ -0,0 +1,34 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ */
8	+ namespace Piwik\DataAccess;
9	+
10	+ use Piwik\Db;
11	+ use Piwik\Common;
12	+
13	+ /**
14	+ * Data Access Object for operations dealing with the log_action table.
15	+ */
16	+ class Actions
17	+ {
18	+ /**
19	+ * Removes a list of actions from the log_action table by ID.
20	+ *
21	+ * @param int[] $idActions
22	+ */
23	+ public function delete($idActions)
24	+ {
25	+ foreach ($idActions as &$id) {
26	+ $id = (int)$id;
27	+ }
28	+
29	+ $table = Common::prefixTable('log_action');
30	+
31	+ $sql = "DELETE FROM $table WHERE idaction IN (" . implode(",", $idActions) . ")";
32	+ Db::query($sql);
33	+ }
34	+ }

app/core/DataAccess/ArchiveSelector.php ADDED Viewed

	@@ -0,0 +1,384 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik\DataAccess;
10	+
11	+ use Exception;
12	+ use Piwik\Archive;
13	+ use Piwik\Archive\Chunk;
14	+ use Piwik\ArchiveProcessor;
15	+ use Piwik\ArchiveProcessor\Rules;
16	+ use Piwik\Common;
17	+ use Piwik\Date;
18	+ use Piwik\Db;
19	+ use Piwik\Period;
20	+ use Piwik\Period\Range;
21	+ use Piwik\Segment;
22	+
23	+ /**
24	+ * Data Access object used to query archives
25	+ *
26	+ * A record in the Database for a given report is defined by
27	+ * - idarchive = unique ID that is associated to all the data of this archive (idsite+period+date)
28	+ * - idsite = the ID of the website
29	+ * - date1 = starting day of the period
30	+ * - date2 = ending day of the period
31	+ * - period = integer that defines the period (day/week/etc.). @see period::getId()
32	+ * - ts_archived = timestamp when the archive was processed (UTC)
33	+ * - name = the name of the report (ex: uniq_visitors or search_keywords_by_search_engines)
34	+ * - value = the actual data (a numeric value, or a blob of compressed serialized data)
35	+ *
36	+ */
37	+ class ArchiveSelector
38	+ {
39	+ const NB_VISITS_RECORD_LOOKED_UP = "nb_visits";
40	+
41	+ const NB_VISITS_CONVERTED_RECORD_LOOKED_UP = "nb_visits_converted";
42	+
43	+ private static function getModel()
44	+ {
45	+ return new Model();
46	+ }
47	+
48	+ /**
49	+ * @param ArchiveProcessor\Parameters $params
50	+ * @param bool $minDatetimeArchiveProcessedUTC deprecated. will be removed in Matomo 4.
51	+ * @return array\|bool
52	+ * @throws Exception
53	+ */
54	+ public static function getArchiveIdAndVisits(ArchiveProcessor\Parameters $params, $minDatetimeArchiveProcessedUTC = false, $includeInvalidated = true)
55	+ {
56	+ $idSite = $params->getSite()->getId();
57	+ $period = $params->getPeriod()->getId();
58	+ $dateStart = $params->getPeriod()->getDateStart();
59	+ $dateStartIso = $dateStart->toString('Y-m-d');
60	+ $dateEndIso = $params->getPeriod()->getDateEnd()->toString('Y-m-d');
61	+
62	+ $numericTable = ArchiveTableCreator::getNumericTable($dateStart);
63	+
64	+ $minDatetimeIsoArchiveProcessedUTC = null;
65	+ if ($minDatetimeArchiveProcessedUTC) {
66	+ $minDatetimeIsoArchiveProcessedUTC = Date::factory($minDatetimeArchiveProcessedUTC)->getDatetime();
67	+ }
68	+
69	+ $requestedPlugin = $params->getRequestedPlugin();
70	+ $segment = $params->getSegment();
71	+ $plugins = array("VisitsSummary", $requestedPlugin);
72	+
73	+ $doneFlags = Rules::getDoneFlags($plugins, $segment);
74	+ $doneFlagValues = Rules::getSelectableDoneFlagValues($includeInvalidated, $params);
75	+
76	+ $results = self::getModel()->getArchiveIdAndVisits($numericTable, $idSite, $period, $dateStartIso, $dateEndIso, $minDatetimeIsoArchiveProcessedUTC, $doneFlags, $doneFlagValues);
77	+
78	+ if (empty($results)) {
79	+ return false;
80	+ }
81	+
82	+ $idArchive = self::getMostRecentIdArchiveFromResults($segment, $requestedPlugin, $results);
83	+
84	+ $idArchiveVisitsSummary = self::getMostRecentIdArchiveFromResults($segment, "VisitsSummary", $results);
85	+
86	+ list($visits, $visitsConverted) = self::getVisitsMetricsFromResults($idArchive, $idArchiveVisitsSummary, $results);
87	+
88	+ if (false === $visits && false === $idArchive) {
89	+ return false;
90	+ }
91	+
92	+ return array($idArchive, $visits, $visitsConverted);
93	+ }
94	+
95	+ protected static function getVisitsMetricsFromResults($idArchive, $idArchiveVisitsSummary, $results)
96	+ {
97	+ $visits = $visitsConverted = false;
98	+ $archiveWithVisitsMetricsWasFound = ($idArchiveVisitsSummary !== false);
99	+
100	+ if ($archiveWithVisitsMetricsWasFound) {
101	+ $visits = $visitsConverted = 0;
102	+ }
103	+
104	+ foreach ($results as $result) {
105	+ if (in_array($result['idarchive'], array($idArchive, $idArchiveVisitsSummary))) {
106	+ $value = (int)$result['value'];
107	+ if (empty($visits)
108	+ && $result['name'] == self::NB_VISITS_RECORD_LOOKED_UP
109	+ ) {
110	+ $visits = $value;
111	+ }
112	+ if (empty($visitsConverted)
113	+ && $result['name'] == self::NB_VISITS_CONVERTED_RECORD_LOOKED_UP
114	+ ) {
115	+ $visitsConverted = $value;
116	+ }
117	+ }
118	+ }
119	+
120	+ return array($visits, $visitsConverted);
121	+ }
122	+
123	+ protected static function getMostRecentIdArchiveFromResults(Segment $segment, $requestedPlugin, $results)
124	+ {
125	+ $idArchive = false;
126	+ $namesRequestedPlugin = Rules::getDoneFlags(array($requestedPlugin), $segment);
127	+
128	+ foreach ($results as $result) {
129	+ if ($idArchive === false
130	+ && in_array($result['name'], $namesRequestedPlugin)
131	+ ) {
132	+ $idArchive = $result['idarchive'];
133	+ break;
134	+ }
135	+ }
136	+
137	+ return $idArchive;
138	+ }
139	+
140	+ /**
141	+ * Queries and returns archive IDs for a set of sites, periods, and a segment.
142	+ *
143	+ * @param array $siteIds
144	+ * @param array $periods
145	+ * @param Segment $segment
146	+ * @param array $plugins List of plugin names for which data is being requested.
147	+ * @param bool $includeInvalidated true to include archives that are DONE_INVALIDATED, false if only DONE_OK.
148	+ * @return array Archive IDs are grouped by archive name and period range, ie,
149	+ * array(
150	+ * 'VisitsSummary.done' => array(
151	+ * '2010-01-01' => array(1,2,3)
152	+ * )
153	+ * )
154	+ * @throws
155	+ */
156	+ public static function getArchiveIds($siteIds, $periods, $segment, $plugins, $includeInvalidated = true)
157	+ {
158	+ if (empty($siteIds)) {
159	+ throw new \Exception("Website IDs could not be read from the request, ie. idSite=");
160	+ }
161	+
162	+ foreach ($siteIds as $index => $siteId) {
163	+ $siteIds[$index] = (int) $siteId;
164	+ }
165	+
166	+ $getArchiveIdsSql = "SELECT idsite, name, date1, date2, MAX(idarchive) as idarchive
167	+ FROM %s
168	+ WHERE idsite IN (" . implode(',', $siteIds) . ")
169	+ AND " . self::getNameCondition($plugins, $segment, $includeInvalidated) . "
170	+ AND %s
171	+ GROUP BY idsite, date1, date2, name";
172	+
173	+ $monthToPeriods = array();
174	+ foreach ($periods as $period) {
175	+ /** @var Period $period */
176	+ if ($period->getDateStart()->isLater(Date::now()->addDay(2))) {
177	+ continue; // avoid creating any archive tables in the future
178	+ }
179	+ $table = ArchiveTableCreator::getNumericTable($period->getDateStart());
180	+ $monthToPeriods[$table][] = $period;
181	+ }
182	+
183	+ $db = Db::get();
184	+
185	+ // for every month within the archive query, select from numeric table
186	+ $result = array();
187	+ foreach ($monthToPeriods as $table => $periods) {
188	+ $firstPeriod = reset($periods);
189	+
190	+ $bind = array();
191	+
192	+ if ($firstPeriod instanceof Range) {
193	+ $dateCondition = "date1 = ? AND date2 = ?";
194	+ $bind[] = $firstPeriod->getDateStart()->toString('Y-m-d');
195	+ $bind[] = $firstPeriod->getDateEnd()->toString('Y-m-d');
196	+ } else {
197	+ // we assume there is no range date in $periods
198	+ $dateCondition = '(';
199	+
200	+ foreach ($periods as $period) {
201	+ if (strlen($dateCondition) > 1) {
202	+ $dateCondition .= ' OR ';
203	+ }
204	+
205	+ $dateCondition .= "(period = ? AND date1 = ? AND date2 = ?)";
206	+ $bind[] = $period->getId();
207	+ $bind[] = $period->getDateStart()->toString('Y-m-d');
208	+ $bind[] = $period->getDateEnd()->toString('Y-m-d');
209	+ }
210	+
211	+ $dateCondition .= ')';
212	+ }
213	+
214	+ $sql = sprintf($getArchiveIdsSql, $table, $dateCondition);
215	+
216	+
217	+ $archiveIds = $db->fetchAll($sql, $bind);
218	+
219	+ // get the archive IDs
220	+ foreach ($archiveIds as $row) {
221	+ //FIXMEA duplicate with Archive.php
222	+ $dateStr = $row['date1'] . ',' . $row['date2'];
223	+
224	+ $result[$row['name']][$dateStr][] = $row['idarchive'];
225	+ }
226	+ }
227	+
228	+ return $result;
229	+ }
230	+
231	+ /**
232	+ * Queries and returns archive data using a set of archive IDs.
233	+ *
234	+ * @param array $archiveIds The IDs of the archives to get data from.
235	+ * @param array $recordNames The names of the data to retrieve (ie, nb_visits, nb_actions, etc.).
236	+ * Note: You CANNOT pass multiple recordnames if $loadAllSubtables=true.
237	+ * @param string $archiveDataType The archive data type (either, 'blob' or 'numeric').
238	+ * @param int\|null\|string $idSubtable null if the root blob should be loaded, an integer if a subtable should be
239	+ * loaded and 'all' if all subtables should be loaded.
240	+ * @return array
241	+ *@throws Exception
242	+ */
243	+ public static function getArchiveData($archiveIds, $recordNames, $archiveDataType, $idSubtable)
244	+ {
245	+ $chunk = new Chunk();
246	+
247	+ $db = Db::get();
248	+
249	+ // create the SQL to select archive data
250	+ $loadAllSubtables = $idSubtable == Archive::ID_SUBTABLE_LOAD_ALL_SUBTABLES;
251	+ if ($loadAllSubtables) {
252	+ $name = reset($recordNames);
253	+
254	+ // select blobs w/ name like "$name_[0-9]+" w/o using RLIKE
255	+ $nameEnd = strlen($name) + 1;
256	+ $nameEndAppendix = $nameEnd + 1;
257	+ $appendix = $chunk->getAppendix();
258	+ $lenAppendix = strlen($appendix);
259	+
260	+ $checkForChunkBlob = "SUBSTRING(name, $nameEnd, $lenAppendix) = '$appendix'";
261	+ $checkForSubtableId = "(SUBSTRING(name, $nameEndAppendix, 1) >= '0'
262	+ AND SUBSTRING(name, $nameEndAppendix, 1) <= '9')";
263	+
264	+ $whereNameIs = "(name = ? OR (name LIKE ? AND ( $checkForChunkBlob OR $checkForSubtableId ) ))";
265	+ $bind = array($name, $name . '%');
266	+ } else {
267	+ if ($idSubtable === null) {
268	+ // select root table or specific record names
269	+ $bind = array_values($recordNames);
270	+ } else {
271	+ // select a subtable id
272	+ $bind = array();
273	+ foreach ($recordNames as $recordName) {
274	+ // to be backwards compatibe we need to look for the exact idSubtable blob and for the chunk
275	+ // that stores the subtables (a chunk stores many blobs in one blob)
276	+ $bind[] = $chunk->getRecordNameForTableId($recordName, $idSubtable);
277	+ $bind[] = self::appendIdSubtable($recordName, $idSubtable);
278	+ }
279	+ }
280	+
281	+ $inNames = Common::getSqlStringFieldsArray($bind);
282	+ $whereNameIs = "name IN ($inNames)";
283	+ }
284	+
285	+ $getValuesSql = "SELECT value, name, idsite, date1, date2, ts_archived
286	+ FROM %s
287	+ WHERE idarchive IN (%s)
288	+ AND " . $whereNameIs;
289	+
290	+ // get data from every table we're querying
291	+ $rows = array();
292	+ foreach ($archiveIds as $period => $ids) {
293	+ if (empty($ids)) {
294	+ throw new Exception("Unexpected: id archive not found for period '$period' '");
295	+ }
296	+
297	+ // $period = "2009-01-04,2009-01-04",
298	+ $date = Date::factory(substr($period, 0, 10));
299	+
300	+ $isNumeric = $archiveDataType == 'numeric';
301	+ if ($isNumeric) {
302	+ $table = ArchiveTableCreator::getNumericTable($date);
303	+ } else {
304	+ $table = ArchiveTableCreator::getBlobTable($date);
305	+ }
306	+
307	+ $sql = sprintf($getValuesSql, $table, implode(',', $ids));
308	+ $dataRows = $db->fetchAll($sql, $bind);
309	+
310	+ foreach ($dataRows as $row) {
311	+ if ($isNumeric) {
312	+ $rows[] = $row;
313	+ } else {
314	+ $row['value'] = self::uncompress($row['value']);
315	+
316	+ if ($chunk->isRecordNameAChunk($row['name'])) {
317	+ self::moveChunkRowToRows($rows, $row, $chunk, $loadAllSubtables, $idSubtable);
318	+ } else {
319	+ $rows[] = $row;
320	+ }
321	+ }
322	+ }
323	+ }
324	+
325	+ return $rows;
326	+ }
327	+
328	+ private static function moveChunkRowToRows(&$rows, $row, Chunk $chunk, $loadAllSubtables, $idSubtable)
329	+ {
330	+ // $blobs = array([subtableID] = [blob of subtableId])
331	+ $blobs = Common::safe_unserialize($row['value']);
332	+
333	+ if (!is_array($blobs)) {
334	+ return;
335	+ }
336	+
337	+ // $rawName = eg 'PluginName_ArchiveName'
338	+ $rawName = $chunk->getRecordNameWithoutChunkAppendix($row['name']);
339	+
340	+ if ($loadAllSubtables) {
341	+ foreach ($blobs as $subtableId => $blob) {
342	+ $row['value'] = $blob;
343	+ $row['name'] = self::appendIdSubtable($rawName, $subtableId);
344	+ $rows[] = $row;
345	+ }
346	+ } elseif (array_key_exists($idSubtable, $blobs)) {
347	+ $row['value'] = $blobs[$idSubtable];
348	+ $row['name'] = self::appendIdSubtable($rawName, $idSubtable);
349	+ $rows[] = $row;
350	+ }
351	+ }
352	+
353	+ public static function appendIdSubtable($recordName, $id)
354	+ {
355	+ return $recordName . "_" . $id;
356	+ }
357	+
358	+ private static function uncompress($data)
359	+ {
360	+ return @gzuncompress($data);
361	+ }
362	+
363	+ /**
364	+ * Returns the SQL condition used to find successfully completed archives that
365	+ * this instance is querying for.
366	+ *
367	+ * @param array $plugins
368	+ * @param Segment $segment
369	+ * @param bool $includeInvalidated
370	+ * @return string
371	+ */
372	+ private static function getNameCondition(array $plugins, Segment $segment, $includeInvalidated = true)
373	+ {
374	+ // the flags used to tell how the archiving process for a specific archive was completed,
375	+ // if it was completed
376	+ $doneFlags = Rules::getDoneFlags($plugins, $segment);
377	+ $allDoneFlags = "'" . implode("','", $doneFlags) . "'";
378	+
379	+ $possibleValues = Rules::getSelectableDoneFlagValues($includeInvalidated);
380	+
381	+ // create the SQL to find archives that are DONE
382	+ return "((name IN ($allDoneFlags)) AND (value IN (" . implode(',', $possibleValues) . ")))";
383	+ }
384	+ }

app/core/DataAccess/ArchiveTableCreator.php ADDED Viewed

	@@ -0,0 +1,123 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+
10	+ namespace Piwik\DataAccess;
11	+
12	+ use Piwik\Common;
13	+ use Piwik\Date;
14	+ use Piwik\DbHelper;
15	+
16	+ class ArchiveTableCreator
17	+ {
18	+ const NUMERIC_TABLE = "numeric";
19	+ const BLOB_TABLE = "blob";
20	+
21	+ public static $tablesAlreadyInstalled = null;
22	+
23	+ public static function getNumericTable(Date $date)
24	+ {
25	+ return self::getTable($date, self::NUMERIC_TABLE);
26	+ }
27	+
28	+ public static function getBlobTable(Date $date)
29	+ {
30	+ return self::getTable($date, self::BLOB_TABLE);
31	+ }
32	+
33	+ protected static function getTable(Date $date, $type)
34	+ {
35	+ $tableNamePrefix = "archive_" . $type;
36	+ $tableName = $tableNamePrefix . "_" . self::getTableMonthFromDate($date);
37	+ $tableName = Common::prefixTable($tableName);
38	+
39	+ self::createArchiveTablesIfAbsent($tableName, $tableNamePrefix);
40	+
41	+ return $tableName;
42	+ }
43	+
44	+ protected static function createArchiveTablesIfAbsent($tableName, $tableNamePrefix)
45	+ {
46	+ if (is_null(self::$tablesAlreadyInstalled)) {
47	+ self::refreshTableList();
48	+ }
49	+
50	+ if (!in_array($tableName, self::$tablesAlreadyInstalled)) {
51	+ self::getModel()->createArchiveTable($tableName, $tableNamePrefix);
52	+ self::$tablesAlreadyInstalled[] = $tableName;
53	+ }
54	+ }
55	+
56	+ private static function getModel()
57	+ {
58	+ return new Model();
59	+ }
60	+
61	+ public static function clear()
62	+ {
63	+ self::$tablesAlreadyInstalled = null;
64	+ }
65	+
66	+ public static function refreshTableList($forceReload = false)
67	+ {
68	+ self::$tablesAlreadyInstalled = DbHelper::getTablesInstalled($forceReload);
69	+ }
70	+
71	+ /**
72	+ * Returns all table names archive_*
73	+ *
74	+ * @param string $type The type of table to return. Either `self::NUMERIC_TABLE` or `self::BLOB_TABLE`.
75	+ * @return array
76	+ */
77	+ public static function getTablesArchivesInstalled($type = null)
78	+ {
79	+ if (is_null(self::$tablesAlreadyInstalled)) {
80	+ self::refreshTableList();
81	+ }
82	+
83	+ if (empty($type)) {
84	+ $tableMatchRegex = '/archive_(numeric\|blob)_/';
85	+ } else {
86	+ $tableMatchRegex = '/archive_' . preg_quote($type) . '_/';
87	+ }
88	+
89	+ $archiveTables = array();
90	+ foreach (self::$tablesAlreadyInstalled as $table) {
91	+ if (preg_match($tableMatchRegex, $table)) {
92	+ $archiveTables[] = $table;
93	+ }
94	+ }
95	+ return $archiveTables;
96	+ }
97	+
98	+ public static function getDateFromTableName($tableName)
99	+ {
100	+ $tableName = Common::unprefixTable($tableName);
101	+ $date = str_replace(array('archive_numeric_', 'archive_blob_'), '', $tableName);
102	+
103	+ return $date;
104	+ }
105	+
106	+ public static function getTableMonthFromDate(Date $date)
107	+ {
108	+ return $date->toString('Y_m');
109	+ }
110	+
111	+ public static function getTypeFromTableName($tableName)
112	+ {
113	+ if (strpos($tableName, 'archive_numeric_') !== false) {
114	+ return self::NUMERIC_TABLE;
115	+ }
116	+
117	+ if (strpos($tableName, 'archive_blob_') !== false) {
118	+ return self::BLOB_TABLE;
119	+ }
120	+
121	+ return false;
122	+ }
123	+ }

app/core/DataAccess/ArchiveTableDao.php ADDED Viewed

	@@ -0,0 +1,90 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ */
8	+
9	+ namespace Piwik\DataAccess;
10	+
11	+ use Piwik\Common;
12	+ use Piwik\Config;
13	+ use Piwik\Db;
14	+
15	+ /**
16	+ * Data Access class for querying numeric & blob archive tables.
17	+ */
18	+ class ArchiveTableDao
19	+ {
20	+ /**
21	+ * Analyzes numeric & blob tables for a single table date (ie, `'2015_01'`) and returns
22	+ * statistics including:
23	+ *
24	+ * - number of archives present
25	+ * - number of invalidated archives
26	+ * - number of temporary archives
27	+ * - number of error archives
28	+ * - number of segment archives
29	+ * - number of numeric rows
30	+ * - number of blob rows
31	+ *
32	+ * @param string $tableDate ie `'2015_01'`
33	+ * @return array
34	+ */
35	+ public function getArchiveTableAnalysis($tableDate)
36	+ {
37	+ $numericQueryEmptyRow = array(
38	+ 'count_archives' => '-',
39	+ 'count_invalidated_archives' => '-',
40	+ 'count_temporary_archives' => '-',
41	+ 'count_error_archives' => '-',
42	+ 'count_segment_archives' => '-',
43	+ 'count_numeric_rows' => '-',
44	+ );
45	+
46	+ $tableDate = str_replace("`", "", $tableDate); // for sanity
47	+
48	+ $numericTable = Common::prefixTable("archive_numeric_$tableDate");
49	+ $blobTable = Common::prefixTable("archive_blob_$tableDate");
50	+
51	+ // query numeric table
52	+ $sql = "SELECT CONCAT_WS('.', idsite, date1, date2, period) AS label,
53	+ SUM(CASE WHEN name LIKE 'done%' THEN 1 ELSE 0 END) AS count_archives,
54	+ SUM(CASE WHEN name LIKE 'done%' AND value = ? THEN 1 ELSE 0 END) AS count_invalidated_archives,
55	+ SUM(CASE WHEN name LIKE 'done%' AND value = ? THEN 1 ELSE 0 END) AS count_temporary_archives,
56	+ SUM(CASE WHEN name LIKE 'done%' AND value = ? THEN 1 ELSE 0 END) AS count_error_archives,
57	+ SUM(CASE WHEN name LIKE 'done%' AND CHAR_LENGTH(name) > 32 THEN 1 ELSE 0 END) AS count_segment_archives,
58	+ SUM(CASE WHEN name NOT LIKE 'done%' THEN 1 ELSE 0 END) AS count_numeric_rows,
59	+ 0 AS count_blob_rows
60	+ FROM `$numericTable`
61	+ GROUP BY idsite, date1, date2, period";
62	+
63	+ $rows = Db::fetchAll($sql, array(ArchiveWriter::DONE_INVALIDATED, ArchiveWriter::DONE_OK_TEMPORARY,
64	+ ArchiveWriter::DONE_ERROR));
65	+
66	+ // index result
67	+ $result = array();
68	+ foreach ($rows as $row) {
69	+ $result[$row['label']] = $row;
70	+ }
71	+
72	+ // query blob table & manually merge results (no FULL OUTER JOIN in mysql)
73	+ $sql = "SELECT CONCAT_WS('.', idsite, date1, date2, period) AS label,
74	+ COUNT(*) AS count_blob_rows,
75	+ SUM(OCTET_LENGTH(value)) AS sum_blob_length
76	+ FROM `$blobTable`
77	+ GROUP BY idsite, date1, date1, period";
78	+
79	+ foreach (Db::fetchAll($sql) as $blobStatsRow) {
80	+ $label = $blobStatsRow['label'];
81	+ if (isset($result[$label])) {
82	+ $result[$label] = array_merge($result[$label], $blobStatsRow);
83	+ } else {
84	+ $result[$label] = $blobStatsRow + $numericQueryEmptyRow;
85	+ }
86	+ }
87	+
88	+ return $result;
89	+ }
90	+ }

app/core/DataAccess/ArchiveWriter.php ADDED Viewed

	@@ -0,0 +1,303 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik\DataAccess;
10	+
11	+ use Exception;
12	+ use Piwik\Archive\Chunk;
13	+ use Piwik\ArchiveProcessor\Rules;
14	+ use Piwik\ArchiveProcessor;
15	+ use Piwik\Db;
16	+ use Piwik\Db\BatchInsert;
17	+
18	+ /**
19	+ * This class is used to create a new Archive.
20	+ * An Archive is a set of reports (numeric and data tables).
21	+ * New data can be inserted in the archive with insertRecord/insertBulkRecords
22	+ */
23	+ class ArchiveWriter
24	+ {
25	+ /**
26	+ * Flag stored at the end of the archiving
27	+ *
28	+ * @var int
29	+ */
30	+ const DONE_OK = 1;
31	+ /**
32	+ * Flag stored at the start of the archiving
33	+ * When requesting an Archive, we make sure that non-finished archive are not considered valid
34	+ *
35	+ * @var int
36	+ */
37	+ const DONE_ERROR = 2;
38	+
39	+ /**
40	+ * Flag indicates the archive is over a period that is not finished, eg. the current day, current week, etc.
41	+ * Archives flagged will be regularly purged from the DB.
42	+ *
43	+ * This flag is deprecated, new archives should not be written as temporary.
44	+ *
45	+ * @var int
46	+ * @deprecated
47	+ */
48	+ const DONE_OK_TEMPORARY = 3;
49	+
50	+ /**
51	+ * Flag indicated that archive is done but was marked as invalid later and needs to be re-processed during next archiving process
52	+ *
53	+ * @var int
54	+ */
55	+ const DONE_INVALIDATED = 4;
56	+
57	+ protected $fields = array('idarchive',
58	+ 'idsite',
59	+ 'date1',
60	+ 'date2',
61	+ 'period',
62	+ 'ts_archived',
63	+ 'name',
64	+ 'value');
65	+
66	+ private $recordsToWriteSpool = array(
67	+ 'numeric' => array(),
68	+ 'blob' => array()
69	+ );
70	+
71	+ const MAX_SPOOL_SIZE = 50;
72	+
73	+ /**
74	+ * ArchiveWriter constructor.
75	+ * @param ArchiveProcessor\Parameters $params
76	+ * @param bool $isArchiveTemporary Deprecated. Has no effect.
77	+ * @throws Exception
78	+ */
79	+ public function __construct(ArchiveProcessor\Parameters $params, $isArchiveTemporary = false)
80	+ {
81	+ $this->idArchive = false;
82	+ $this->idSite = $params->getSite()->getId();
83	+ $this->segment = $params->getSegment();
84	+ $this->period = $params->getPeriod();
85	+
86	+ $idSites = array($this->idSite);
87	+ $this->doneFlag = Rules::getDoneStringFlagFor($idSites, $this->segment, $this->period->getLabel(), $params->getRequestedPlugin());
88	+
89	+ $this->dateStart = $this->period->getDateStart();
90	+ }
91	+
92	+ /**
93	+ * @param string $name
94	+ * @param string\|string[] $values A blob string or an array of blob strings. If an array
95	+ * is used, the first element in the array will be inserted
96	+ * with the `$name` name. The others will be splitted into chunks. All subtables
97	+ * within one chunk will be serialized as an array where the index is the
98	+ * subtableId.
99	+ */
100	+ public function insertBlobRecord($name, $values)
101	+ {
102	+ if (is_array($values)) {
103	+
104	+ if (isset($values[0])) {
105	+ // we always store the root table in a single blob for fast access
106	+ $this->insertRecord($name, $this->compress($values[0]));
107	+ unset($values[0]);
108	+ }
109	+
110	+ if (!empty($values)) {
111	+ // we move all subtables into chunks
112	+ $chunk = new Chunk();
113	+ $chunks = $chunk->moveArchiveBlobsIntoChunks($name, $values);
114	+ foreach ($chunks as $index => $subtables) {
115	+ $this->insertRecord($index, $this->compress(serialize($subtables)));
116	+ }
117	+ }
118	+ } else {
119	+ $values = $this->compress($values);
120	+ $this->insertRecord($name, $values);
121	+ }
122	+ }
123	+
124	+ public function getIdArchive()
125	+ {
126	+ if ($this->idArchive === false) {
127	+ throw new Exception("Must call allocateNewArchiveId() first");
128	+ }
129	+
130	+ return $this->idArchive;
131	+ }
132	+
133	+ public function initNewArchive()
134	+ {
135	+ $this->allocateNewArchiveId();
136	+ $this->logArchiveStatusAsIncomplete();
137	+ }
138	+
139	+ public function finalizeArchive()
140	+ {
141	+ $this->flushSpools();
142	+
143	+ $numericTable = $this->getTableNumeric();
144	+ $idArchive = $this->getIdArchive();
145	+
146	+ $this->getModel()->updateArchiveStatus($numericTable, $idArchive, $this->doneFlag, self::DONE_OK);
147	+ }
148	+
149	+ protected function compress($data)
150	+ {
151	+ if (Db::get()->hasBlobDataType()) {
152	+ return gzcompress($data);
153	+ }
154	+
155	+ return $data;
156	+ }
157	+
158	+ protected function allocateNewArchiveId()
159	+ {
160	+ $numericTable = $this->getTableNumeric();
161	+
162	+ $this->idArchive = $this->getModel()->allocateNewArchiveId($numericTable);
163	+ return $this->idArchive;
164	+ }
165	+
166	+ private function getModel()
167	+ {
168	+ return new Model();
169	+ }
170	+
171	+ protected function logArchiveStatusAsIncomplete()
172	+ {
173	+ $this->insertRecord($this->doneFlag, self::DONE_ERROR);
174	+ }
175	+
176	+ private function batchInsertSpool($valueType)
177	+ {
178	+ $records = $this->recordsToWriteSpool[$valueType];
179	+
180	+ $bindSql = $this->getInsertRecordBind();
181	+ $values = array();
182	+
183	+ $valueSeen = false;
184	+ foreach ($records as $record) {
185	+ // don't record zero
186	+ if (empty($record[1])) {
187	+ continue;
188	+ }
189	+
190	+ $bind = $bindSql;
191	+ $bind[] = $record[0]; // name
192	+ $bind[] = $record[1]; // value
193	+ $values[] = $bind;
194	+
195	+ $valueSeen = $record[1];
196	+ }
197	+
198	+ if (empty($values)) {
199	+ return true;
200	+ }
201	+
202	+ $tableName = $this->getTableNameToInsert($valueSeen);
203	+ $fields = $this->getInsertFields();
204	+
205	+ // For numeric records it's faster to do the insert directly; for blobs the data infile is better
206	+ if ($valueType == 'numeric') {
207	+ BatchInsert::tableInsertBatchSql($tableName, $fields, $values);
208	+ } else {
209	+ BatchInsert::tableInsertBatch($tableName, $fields, $values, $throwException = false, $charset = 'latin1');
210	+ }
211	+
212	+ return true;
213	+ }
214	+
215	+ /**
216	+ * Inserts a record in the right table (either NUMERIC or BLOB)
217	+ *
218	+ * @param string $name
219	+ * @param mixed $value
220	+ *
221	+ * @return bool
222	+ */
223	+ public function insertRecord($name, $value)
224	+ {
225	+ if ($this->isRecordZero($value)) {
226	+ return false;
227	+ }
228	+
229	+ $valueType = $this->isRecordNumeric($value) ? 'numeric' : 'blob';
230	+ $this->recordsToWriteSpool[$valueType][] = array(
231	+ 0 => $name,
232	+ 1 => $value
233	+ );
234	+
235	+ if (count($this->recordsToWriteSpool[$valueType]) >= self::MAX_SPOOL_SIZE) {
236	+ $this->flushSpool($valueType);
237	+ }
238	+
239	+ return true;
240	+ }
241	+
242	+ public function flushSpools()
243	+ {
244	+ $this->flushSpool('numeric');
245	+ $this->flushSpool('blob');
246	+ }
247	+
248	+ private function flushSpool($valueType)
249	+ {
250	+ $numRecords = count($this->recordsToWriteSpool[$valueType]);
251	+
252	+ if ($numRecords > 1) {
253	+ $this->batchInsertSpool($valueType);
254	+ } elseif ($numRecords == 1) {
255	+ list($name, $value) = $this->recordsToWriteSpool[$valueType][0];
256	+ $tableName = $this->getTableNameToInsert($value);
257	+ $fields = $this->getInsertFields();
258	+ $record = $this->getInsertRecordBind();
259	+
260	+ $this->getModel()->insertRecord($tableName, $fields, $record, $name, $value);
261	+ }
262	+ $this->recordsToWriteSpool[$valueType] = array();
263	+ }
264	+
265	+ protected function getInsertRecordBind()
266	+ {
267	+ return array($this->getIdArchive(),
268	+ $this->idSite,
269	+ $this->dateStart->toString('Y-m-d'),
270	+ $this->period->getDateEnd()->toString('Y-m-d'),
271	+ $this->period->getId(),
272	+ date("Y-m-d H:i:s"));
273	+ }
274	+
275	+ protected function getTableNameToInsert($value)
276	+ {
277	+ if ($this->isRecordNumeric($value)) {
278	+ return $this->getTableNumeric();
279	+ }
280	+
281	+ return ArchiveTableCreator::getBlobTable($this->dateStart);
282	+ }
283	+
284	+ protected function getTableNumeric()
285	+ {
286	+ return ArchiveTableCreator::getNumericTable($this->dateStart);
287	+ }
288	+
289	+ protected function getInsertFields()
290	+ {
291	+ return $this->fields;
292	+ }
293	+
294	+ protected function isRecordZero($value)
295	+ {
296	+ return ($value === '0' \|\| $value === false \|\| $value === 0 \|\| $value === 0.0);
297	+ }
298	+
299	+ private function isRecordNumeric($value)
300	+ {
301	+ return is_numeric($value);
302	+ }
303	+ }

app/core/DataAccess/ArchivingDbAdapter.php ADDED Viewed

	@@ -0,0 +1,107 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+
10	+ namespace Piwik\DataAccess;
11	+
12	+ use Piwik\Concurrency\Lock;
13	+ use Piwik\Db\AdapterInterface;
14	+ use Psr\Log\LoggerInterface;
15	+
16	+ class ArchivingDbAdapter
17	+ {
18	+ /**
19	+ * @var AdapterInterface\|\Zend_Db_Adapter_Abstract
20	+ */
21	+ private $wrapped;
22	+
23	+ /**
24	+ * @var Lock
25	+ */
26	+ private $archivingLock;
27	+
28	+ /**
29	+ * @var LoggerInterface
30	+ */
31	+ private $logger;
32	+
33	+ public function __construct($wrapped, Lock $archivingLock = null, LoggerInterface $logger = null)
34	+ {
35	+ $this->wrapped = $wrapped;
36	+ $this->archivingLock = $archivingLock;
37	+ $this->logger = $logger;
38	+ }
39	+
40	+ public function __call($name, $arguments)
41	+ {
42	+ return call_user_func_array([$this->wrapped, $name], $arguments);
43	+ }
44	+
45	+ public function exec($sql)
46	+ {
47	+ $this->reexpireLock();
48	+ $this->logSql($sql);
49	+
50	+ return call_user_func_array([$this->wrapped, __FUNCTION__], func_get_args());
51	+ }
52	+
53	+ public function query($sql)
54	+ {
55	+ $this->reexpireLock();
56	+ $this->logSql($sql);
57	+
58	+ return call_user_func_array([$this->wrapped, __FUNCTION__], func_get_args());
59	+ }
60	+
61	+ public function fetchAll($sql)
62	+ {
63	+ $this->reexpireLock();
64	+ $this->logSql($sql);
65	+
66	+ return call_user_func_array([$this->wrapped, __FUNCTION__], func_get_args());
67	+ }
68	+
69	+ public function fetchRow($sql)
70	+ {
71	+ $this->reexpireLock();
72	+ $this->logSql($sql);
73	+
74	+ return call_user_func_array([$this->wrapped, __FUNCTION__], func_get_args());
75	+ }
76	+
77	+ public function fetchOne($sql)
78	+ {
79	+ $this->reexpireLock();
80	+ $this->logSql($sql);
81	+
82	+ return call_user_func_array([$this->wrapped, __FUNCTION__], func_get_args());
83	+ }
84	+
85	+ public function fetchAssoc($sql)
86	+ {
87	+ $this->reexpireLock();
88	+ $this->logSql($sql);
89	+
90	+ return call_user_func_array([$this->wrapped, __FUNCTION__], func_get_args());
91	+ }
92	+
93	+ private function logSql($sql)
94	+ {
95	+ // Log on DEBUG level all SQL archiving queries
96	+ if ($this->logger) {
97	+ $this->logger->debug($sql);
98	+ }
99	+ }
100	+
101	+ private function reexpireLock()
102	+ {
103	+ if ($this->archivingLock) {
104	+ $this->archivingLock->reexpireLock();
105	+ }
106	+ }
107	+ }

app/core/DataAccess/LogAggregator.php ADDED Viewed

	@@ -0,0 +1,1172 @@


1	+ <?php
2	+ /**
3	+ * Piwik - free/libre analytics platform
4	+ *
5	+ * @link https://matomo.org
6	+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	+ *
8	+ */
9	+ namespace Piwik\DataAccess;
10	+
11	+ use Piwik\ArchiveProcessor\ArchivingStatus;
12	+ use Piwik\ArchiveProcessor\Parameters;
13	+ use Piwik\Common;
14	+ use Piwik\Concurrency\Lock;
15	+ use Piwik\Config;
16	+ use Piwik\Container\StaticContainer;
17	+ use Piwik\DataArray;
18	+ use Piwik\Date;
19	+ use Piwik\Db;
20	+ use Piwik\DbHelper;
21	+ use Piwik\Metrics;
22	+ use Piwik\Period;
23	+ use Piwik\Piwik;
24	+ use Piwik\Plugin\LogTablesProvider;
25	+ use Piwik\Segment;
26	+ use Piwik\Segment\SegmentExpression;
27	+ use Piwik\Tracker\GoalManager;
28	+ use Psr\Log\LoggerInterface;
29	+
30	+ /**
31	+ * Contains methods that calculate metrics by aggregating log data (visits, actions, conversions,
32	+ * ecommerce items).
33	+ *
34	+ * You can use the methods in this class within {@link Piwik\Plugin\Archiver Archiver} descendants
35	+ * to aggregate log data without having to write SQL queries.
36	+ *
37	+ * ### Aggregation Dimension
38	+ *
39	+ * All aggregation methods accept a dimension parameter. These parameters are important as
40	+ * they control how rows in a table are aggregated together.
41	+ *
42	+ * A _dimension_ is just a table column. Rows that have the same values for these columns are
43	+ * aggregated together. The result of these aggregations is a set of metrics for every recorded value
44	+ * of a dimension.
45	+ *
46	+ * _Note: A dimension is essentially the same as a GROUP BY field._
47	+ *
48	+ * ### Examples
49	+ *
50	+ * Aggregating visit data
51	+ *
52	+ * $archiveProcessor = // ...
53	+ * $logAggregator = $archiveProcessor->getLogAggregator();
54	+ *
55	+ * // get metrics for every used browser language of all visits by returning visitors
56	+ * $query = $logAggregator->queryVisitsByDimension(
57	+ * $dimensions = array('log_visit.location_browser_lang'),
58	+ * $where = 'log_visit.visitor_returning = 1',
59	+ *
60	+ * // also count visits for each browser language that are not located in the US
61	+ * $additionalSelects = array('sum(case when log_visit.location_country <> 'us' then 1 else 0 end) as nonus'),
62	+ *
63	+ * // we're only interested in visits, unique visitors & actions, so don't waste time calculating anything else
64	+ * $metrics = array(Metrics::INDEX_NB_UNIQ_VISITORS, Metrics::INDEX_NB_VISITS, Metrics::INDEX_NB_ACTIONS),
65	+ * );
66	+ * if ($query === false) {
67	+ * return;
68	+ * }
69	+ *
70	+ * while ($row = $query->fetch()) {
71	+ * $uniqueVisitors = $row[Metrics::INDEX_NB_UNIQ_VISITORS];
72	+ * $visits = $row[Metrics::INDEX_NB_VISITS];
73	+ * $actions = $row[Metrics::INDEX_NB_ACTIONS];
74	+ *
75	+ * // ... do something w/ calculated metrics ...
76	+ * }
77	+ *
78	+ * Aggregating conversion data
79	+ *
80	+ * $archiveProcessor = // ...
81	+ * $logAggregator = $archiveProcessor->getLogAggregator();
82	+ *
83	+ * // get metrics for ecommerce conversions for each country
84	+ * $query = $logAggregator->queryConversionsByDimension(
85	+ * $dimensions = array('log_conversion.location_country'),
86	+ * $where = 'log_conversion.idgoal = 0', // 0 is the special ecommerceOrder idGoal value in the table
87	+ *
88	+ * // also calculate average tax and max shipping per country
89	+ * $additionalSelects = array(
90	+ * 'AVG(log_conversion.revenue_tax) as avg_tax',
91	+ * 'MAX(log_conversion.revenue_shipping) as max_shipping'
92	+ * )
93	+ * );
94	+ * if ($query === false) {
95	+ * return;
96	+ * }
97	+ *
98	+ * while ($row = $query->fetch()) {
99	+ * $country = $row['location_country'];
100	+ * $numEcommerceSales = $row[Metrics::INDEX_GOAL_NB_CONVERSIONS];
101	+ * $numVisitsWithEcommerceSales = $row[Metrics::INDEX_GOAL_NB_VISITS_CONVERTED];
102	+ * $avgTaxForCountry = $row['avg_tax'];
103	+ * $maxShippingForCountry = $row['max_shipping'];
104	+ *
105	+ * // ... do something with aggregated data ...
106	+ * }
107	+ */
108	+ class LogAggregator
109	+ {
110	+ const LOG_VISIT_TABLE = 'log_visit';
111	+
112	+ const LOG_ACTIONS_TABLE = 'log_link_visit_action';
113	+
114	+ const LOG_CONVERSION_TABLE = "log_conversion";
115	+
116	+ const REVENUE_SUBTOTAL_FIELD = 'revenue_subtotal';
117	+
118	+ const REVENUE_TAX_FIELD = 'revenue_tax';
119	+
120	+ const REVENUE_SHIPPING_FIELD = 'revenue_shipping';
121	+
122	+ const REVENUE_DISCOUNT_FIELD = 'revenue_discount';
123	+
124	+ const TOTAL_REVENUE_FIELD = 'revenue';
125	+
126	+ const ITEMS_COUNT_FIELD = "items";
127	+
128	+ const CONVERSION_DATETIME_FIELD = "server_time";
129	+
130	+ const ACTION_DATETIME_FIELD = "server_time";
131	+
132	+ const VISIT_DATETIME_FIELD = 'visit_last_action_time';
133	+
134	+ const IDGOAL_FIELD = 'idgoal';
135	+
136	+ const FIELDS_SEPARATOR = ", \n\t\t\t";
137	+
138	+ const LOG_TABLE_SEGMENT_TEMPORARY_PREFIX = 'logtmpsegment';
139	+
140	+ /** @var \Piwik\Date */
141	+ protected $dateStart;
142	+
143	+ /** @var \Piwik\Date */
144	+ protected $dateEnd;
145	+
146	+ /** @var int[] */
147	+ protected $sites;
148	+
149	+ /** @var \Piwik\Segment */
150	+ protected $segment;
151	+
152	+ /**
153	+ * @var string
154	+ */
155	+ private $queryOriginHint = '';
156	+
157	+ /**
158	+ * @var LoggerInterface
159	+ */
160	+ private $logger;
161	+
162	+ /**
163	+ * @var bool
164	+ */
165	+ private $isRootArchiveRequest;
166	+
167	+ /**
168	+ * @var bool
169	+ */
170	+ private $allowUsageSegmentCache = false;
171	+
172	+ /**
173	+ * Constructor.
174	+ *
175	+ * @param \Piwik\ArchiveProcessor\Parameters $params
176	+ */
177	+ public function __construct(Parameters $params, LoggerInterface $logger = null)
178	+ {
179	+ $this->dateStart = $params->getDateTimeStart();
180	+ $this->dateEnd = $params->getDateTimeEnd();
181	+ $this->segment = $params->getSegment();
182	+ $this->sites = $params->getIdSites();
183	+ $this->isRootArchiveRequest = $params->isRootArchiveRequest();
184	+ $this->logger = $logger ?: StaticContainer::get('Psr\Log\LoggerInterface');
185	+ }
186	+
187	+ public function getSegment()
188	+ {
189	+ return $this->segment;
190	+ }
191	+
192	+ public function setQueryOriginHint($nameOfOrigiin)
193	+ {
194	+ $this->queryOriginHint = $nameOfOrigiin;
195	+ }
196	+
197	+ public function getSegmentTmpTableName()
198	+ {
199	+ $bind = $this->getGeneralQueryBindParams();
200	+ $tableName = self::LOG_TABLE_SEGMENT_TEMPORARY_PREFIX . md5(json_encode($bind) . $this->segment->getString());
201	+
202	+ $lengthPrefix = Common::mb_strlen(Common::prefixTable(''));
203	+ $maxLength = Db\Schema\Mysql::MAX_TABLE_NAME_LENGTH - $lengthPrefix;
204	+
205	+ return Common::mb_substr($tableName, 0, $maxLength);
206	+ }
207	+
208	+ public function cleanup()
209	+ {
210	+ if (!$this->segment->isEmpty() && $this->isSegmentCacheEnabled()) {
211	+ $segmentTable = $this->getSegmentTmpTableName();
212	+ $segmentTable = Common::prefixTable($segmentTable);
213	+
214	+ if ($this->doesSegmentTableExist($segmentTable)) {
215	+ // safety in case an older MySQL version is used that does not drop table at the end of the connection
216	+ // automatically. also helps us release disk space/memory earlier when multiple segments are archived
217	+ $this->getDb()->query('DROP TEMPORARY TABLE IF EXISTS ' . $segmentTable);
218	+ }
219	+
220	+ $logTablesProvider = $this->getLogTableProvider();
221	+ if ($logTablesProvider->getLogTable($segmentTable)) {
222	+ $logTablesProvider->setTempTable(null); // no longer available
223	+ }
224	+ }
225	+ }
226	+
227	+ private function doesSegmentTableExist($segmentTablePrefixed)
228	+ {
229	+ try {
230	+ // using DROP TABLE IF EXISTS would not work on a DB reader if the table doesn't exist...
231	+ $this->getDb()->fetchOne('SELECT 1 FROM ' . $segmentTablePrefixed . ' LIMIT 1');
232	+ $tableExists = true;
233	+ } catch (\Exception $e) {
234	+ $tableExists = false;
235	+ }
236	+
237	+ return $tableExists;
238	+ }
239	+
240	+ private function isSegmentCacheEnabled()
241	+ {
242	+ if (!$this->allowUsageSegmentCache) {
243	+ return false;
244	+ }
245	+
246	+ $config = Config::getInstance();
247	+ $general = $config->General;
248	+ return !empty($general['enable_segments_cache']);
249	+ }
250	+
251	+ public function allowUsageSegmentCache()
252	+ {
253	+ $this->allowUsageSegmentCache = true;
254	+ }
255	+
256	+ private function getLogTableProvider()
257	+ {
258	+ return StaticContainer::get(LogTablesProvider::class);
259	+ }
260	+
261	+ private function createTemporaryTable($unprefixedSegmentTableName, $segmentSelectSql, $segmentSelectBind)
262	+ {
263	+ $table = Common::prefixTable($unprefixedSegmentTableName);
264	+
265	+ if ($this->doesSegmentTableExist($table)) {
266	+ return; // no need to create the table, it was already created... better to have a select vs unneeded create table
267	+ }
268	+
269	+ $engine = '';
270	+ if (defined('PIWIK_TEST_MODE') && PIWIK_TEST_MODE) {
271	+ $engine = 'ENGINE=MEMORY';
272	+ }
273	+ $createTableSql = 'CREATE TEMPORARY TABLE ' . $table . ' (idvisit BIGINT(10) UNSIGNED NOT NULL) ' . $engine;
274	+ // we do not insert the data right away using create temporary table ... select ...
275	+ // to avoid metadata lock see eg https://www.percona.com/blog/2018/01/10/why-avoid-create-table-as-select-statement/
276	+
277	+ $readerDb = Db::getReader();
278	+ try {
279	+ $readerDb->query($createTableSql);
280	+ } catch (\Exception $e) {
281	+ if ($readerDb->isErrNo($e, \Piwik\Updater\Migration\Db::ERROR_CODE_TABLE_EXISTS)) {
282	+ return;
283	+ }
284	+ throw $e;
285	+ }
286	+
287	+ $transactionLevel = new Db\TransactionLevel($readerDb);
288	+ $canSetTransactionLevel = $transactionLevel->canLikelySetTransactionLevel();
289	+
290	+ if ($canSetTransactionLevel) {
291	+ // i know this could be shortened to one if or one line but I want to make sure this line where we
292	+ // set uncomitted is easily noticable in the code as it could be missed quite easily otherwise
293	+ // we set uncommitted so we don't make the INSERT INTO... SELECT... locking ... we do not want to lock
294	+ // eg the visits table
295	+ if (!$transactionLevel->setUncommitted()) {
296	+ $canSetTransactionLevel = false;
297	+ }
298	+ }
299	+
300	+ if (!$canSetTransactionLevel) {
301	+ // transaction level doesn't work... we're instead executing the select individually and then insert the data
302	+ // this uses more memory but at least is not locking
303	+ $all = $readerDb->fetchAll($segmentSelectSql, $segmentSelectBind);
304	+ if (!empty($all)) {
305	+ // we're not using batchinsert since this would not support the reader DB.
306	+ $readerDb->query('INSERT INTO ' . $table . ' VALUES ('.implode('),(', array_column($all, 'idvisit')).')');
307	+ }
308	+ return;
309	+ }
310	+
311	+ $insertIntoStatement = 'INSERT INTO ' . $table . ' (idvisit) ' . $segmentSelectSql;
312	+ $readerDb->query($insertIntoStatement, $segmentSelectBind);
313	+
314	+ $transactionLevel->restorePreviousStatus();
315	+ }
316	+
317	+ public function generateQuery($select, $from, $where, $groupBy, $orderBy, $limit = 0, $offset = 0)
318	+ {
319	+ $segment = $this->segment;
320	+ $bind = $this->getGeneralQueryBindParams();
321	+
322	+ if (!$this->segment->isEmpty() && $this->isSegmentCacheEnabled()) {
323	+ // here we create the TMP table and apply the segment including the datetime and the requested idsite
324	+ // at the end we generated query will no longer need to apply the datetime/idsite and segment
325	+ $segment = new Segment('', $this->sites);
326	+
327	+ $segmentTable = $this->getSegmentTmpTableName();
328	+
329	+ $segmentWhere = $this->getWhereStatement('log_visit', 'visit_last_action_time');
330	+ $segmentBind = $this->getGeneralQueryBindParams();
331	+
332	+ $logQueryBuilder = StaticContainer::get('Piwik\DataAccess\LogQueryBuilder');
333	+ $forceGroupByBackup = $logQueryBuilder->getForcedInnerGroupBySubselect();
334	+ $logQueryBuilder->forceInnerGroupBySubselect(LogQueryBuilder::FORCE_INNER_GROUP_BY_NO_SUBSELECT);
335	+ $segmentSql = $this->segment->getSelectQuery('distinct log_visit.idvisit as idvisit', 'log_visit', $segmentWhere, $segmentBind, 'log_visit.idvisit ASC');
336	+ $logQueryBuilder->forceInnerGroupBySubselect($forceGroupByBackup);
337	+
338	+ $this->createTemporaryTable($segmentTable, $segmentSql['sql'], $segmentSql['bind']);
339	+
340	+ if (!is_array($from)) {
341	+ $from = array($segmentTable, $from);
342	+ } else {
343	+ array_unshift($from, $segmentTable);
344	+ }
345	+
346	+ $logTablesProvider = $this->getLogTableProvider();
347	+ $logTablesProvider->setTempTable(new LogTableTemporary($segmentTable));
348	+
349	+ foreach ($logTablesProvider->getAllLogTables() as $logTable) {
350	+ if ($logTable->getDateTimeColumn()) {
351	+ $whereTest = $this->getWhereStatement($logTable->getName(), $logTable->getDateTimeColumn());
352	+ if (strpos($where, $whereTest) === 0) {
353	+ // we don't need to apply the where statement again as it would have been applied already
354	+ // in the temporary table... instead it should join the tables through the idvisit index
355	+ $where = ltrim(str_replace($whereTest, '', $where));
356	+ if (stripos($where, 'and ') === 0) {
357	+ $where = substr($where, strlen('and '));
358	+ }
359	+ $bind = array();
360	+ break;
361	+ }
362	+ }
363	+
364	+ }
365	+
366	+ }
367	+
368	+ $query = $segment->getSelectQuery($select, $from, $where, $bind, $orderBy, $groupBy, $limit, $offset);
369	+
370	+ $select = 'SELECT';
371	+ if ($this->queryOriginHint && is_array($query) && 0 === strpos(trim($query['sql']), $select)) {
372	+ $query['sql'] = trim($query['sql']);
373	+ $query['sql'] = 'SELECT /* ' . $this->queryOriginHint . ' */' . substr($query['sql'], strlen($select));
374	+ }
375	+
376	+ if (!$this->getSegment()->isEmpty() && is_array($query) && 0 === strpos(trim($query['sql']), $select)) {
377	+ $query['sql'] = trim($query['sql']);
378	+ $query['sql'] = 'SELECT /* ' . $this->dateStart->toString() . ',' . $this->dateEnd->toString() . ' sites ' . implode(',', array_map('intval', $this->sites)) . ' segmenthash ' . $this->getSegment()->getHash(). ' */' . substr($query['sql'], strlen($select));
379	+ }
380	+
381	+ return $query;
382	+ }
383	+
384	+ protected function getVisitsMetricFields()
385	+ {
386	+ return array(
387	+ Metrics::INDEX_NB_UNIQ_VISITORS => "count(distinct " . self::LOG_VISIT_TABLE . ".idvisitor)",
388	+ Metrics::INDEX_NB_UNIQ_FINGERPRINTS => "count(distinct " . self::LOG_VISIT_TABLE . ".config_id)",
389	+ Metrics::INDEX_NB_VISITS => "count(*)",
390	+ Metrics::INDEX_NB_ACTIONS => "sum(" . self::LOG_VISIT_TABLE . ".visit_total_actions)",
391	+ Metrics::INDEX_MAX_ACTIONS => "max(" . self::LOG_VISIT_TABLE . ".visit_total_actions)",
392	+ Metrics::INDEX_SUM_VISIT_LENGTH => "sum(" . self::LOG_VISIT_TABLE . ".visit_total_time)",
393	+ Metrics::INDEX_BOUNCE_COUNT => "sum(case " . self::LOG_VISIT_TABLE . ".visit_total_actions when 1 then 1 when 0 then 1 else 0 end)",
394	+ Metrics::INDEX_NB_VISITS_CONVERTED => "sum(case " . self::LOG_VISIT_TABLE . ".visit_goal_converted when 1 then 1 else 0 end)",
395	+ Metrics::INDEX_NB_USERS => "count(distinct " . self::LOG_VISIT_TABLE . ".user_id)",
396	+ );
397	+ }
398	+
399	+ public static function getConversionsMetricFields()
400	+ {
401	+ return array(
402	+ Metrics::INDEX_GOAL_NB_CONVERSIONS => "count(*)",
403	+ Metrics::INDEX_GOAL_NB_VISITS_CONVERTED => "count(distinct " . self::LOG_CONVERSION_TABLE . ".idvisit)",
404	+ Metrics::INDEX_GOAL_REVENUE => self::getSqlConversionRevenueSum(self::TOTAL_REVENUE_FIELD),
405	+ Metrics::INDEX_GOAL_ECOMMERCE_REVENUE_SUBTOTAL => self::getSqlConversionRevenueSum(self::REVENUE_SUBTOTAL_FIELD),
406	+ Metrics::INDEX_GOAL_ECOMMERCE_REVENUE_TAX => self::getSqlConversionRevenueSum(self::REVENUE_TAX_FIELD),
407	+ Metrics::INDEX_GOAL_ECOMMERCE_REVENUE_SHIPPING => self::getSqlConversionRevenueSum(self::REVENUE_SHIPPING_FIELD),
408	+ Metrics::INDEX_GOAL_ECOMMERCE_REVENUE_DISCOUNT => self::getSqlConversionRevenueSum(self::REVENUE_DISCOUNT_FIELD),
409	+ Metrics::INDEX_GOAL_ECOMMERCE_ITEMS => "SUM(" . self::LOG_CONVERSION_TABLE . "." . self::ITEMS_COUNT_FIELD . ")",
410	+ );
411	+ }
412	+
413	+ private static function getSqlConversionRevenueSum($field)
414	+ {
415	+ return self::getSqlRevenue('SUM(' . self::LOG_CONVERSION_TABLE . '.' . $field . ')');
416	+ }
417	+
418	+ public static function getSqlRevenue($field)
419	+ {
420	+ return "ROUND(" . $field . "," . GoalManager::REVENUE_PRECISION . ")";
421	+ }
422	+
423	+ /**
424	+ * Helper function that returns an array with common metrics for a given log_visit field distinct values.
425	+ *
426	+ * The statistics returned are:
427	+ * - number of unique visitors
428	+ * - number of visits
429	+ * - number of actions
430	+ * - maximum number of action for a visit
431	+ * - sum of the visits' length in sec
432	+ * - count of bouncing visits (visits with one page view)
433	+ *
434	+ * For example if $dimension = 'config_os' it will return the statistics for every distinct Operating systems
435	+ * The returned array will have a row per distinct operating systems,
436	+ * and a column per stat (nb of visits, max actions, etc)
437	+ *
438	+ * 'label' Metrics::INDEX_NB_UNIQ_VISITORS Metrics::INDEX_NB_VISITS etc.
439	+ * Linux 27 66 ...
440	+ * Windows XP 12 ...
441	+ * Mac OS 15 36 ...
442	+ *
443	+ * @param string $dimension Table log_visit field name to be use to compute common stats
444	+ * @return DataArray
445	+ */
446	+ public function getMetricsFromVisitByDimension($dimension)
447	+ {
448	+ if (!is_array($dimension)) {
449	+ $dimension = array($dimension);
450	+ }
451	+ if (count($dimension) == 1) {
452	+ $dimension = array("label" => reset($dimension));
453	+ }
454	+ $query = $this->queryVisitsByDimension($dimension);
455	+ $metrics = new DataArray();
456	+ while ($row = $query->fetch()) {
457	+ $metrics->sumMetricsVisits($row["label"], $row);
458	+ }
459	+ return $metrics;
460	+ }
461	+
462	+ /**
463	+ * Executes and returns a query aggregating visit logs, optionally grouping by some dimension. Returns
464	+ * a DB statement that can be used to iterate over the result
465	+ *
466	+ * Result Set
467	+ *
468	+ * The following columns are in each row of the result set:
469	+ *
470	+ * - {@link Piwik\Metrics::INDEX_NB_UNIQ_VISITORS}: The total number of unique visitors in this group
471	+ * of aggregated visits.
472	+ * - {@link Piwik\Metrics::INDEX_NB_VISITS}: The total number of visits aggregated.
473	+ * - {@link Piwik\Metrics::INDEX_NB_ACTIONS}: The total number of actions performed in this group of
474	+ * aggregated visits.
475	+ * - {@link Piwik\Metrics::INDEX_MAX_ACTIONS}: The maximum actions perfomred in one visit for this group of
476	+ * visits.
477	+ * - {@link Piwik\Metrics::INDEX_SUM_VISIT_LENGTH}: The total amount of time spent on the site for this
478	+ * group of visits.
479	+ * - {@link Piwik\Metrics::INDEX_BOUNCE_COUNT}: The total number of bounced visits in this group of
480	+ * visits.
481	+ * - {@link Piwik\Metrics::INDEX_NB_VISITS_CONVERTED}: The total number of visits for which at least one
482	+ * conversion occurred, for this group of visits.
483	+ *
484	+ * Additional data can be selected by setting the `$additionalSelects` parameter.
485	+ *
486	+ * _Note: The metrics returned by this query can be customized by the `$metrics` parameter._
487	+ *
488	+ * @param array\|string $dimensions `SELECT` fields (or just one field) that will be grouped by,
489	+ * eg, `'referrer_name'` or `array('referrer_name', 'referrer_keyword')`.
490	+ * The metrics retrieved from the query will be specific to combinations
491	+ * of these fields. So if `array('referrer_name', 'referrer_keyword')`
492	+ * is supplied, the query will aggregate visits for each referrer/keyword
493	+ * combination.
494	+ * @param bool\|string $where Additional condition for the `WHERE` clause. Can be used to filter
495	+ * the set of visits that are considered for aggregation.
496	+ * @param array $additionalSelects Additional `SELECT` fields that are not included in the group by
497	+ * clause. These can be aggregate expressions, eg, `SUM(somecol)`.
498	+ * @param bool\|array $metrics The set of metrics to calculate and return. If false, the query will select
499	+ * all of them. The following values can be used:
500	+ *
501	+ * - {@link Piwik\Metrics::INDEX_NB_UNIQ_VISITORS}
502	+ * - {@link Piwik\Metrics::INDEX_NB_VISITS}
503	+ * - {@link Piwik\Metrics::INDEX_NB_ACTIONS}
504	+ * - {@link Piwik\Metrics::INDEX_MAX_ACTIONS}
505	+ * - {@link Piwik\Metrics::INDEX_SUM_VISIT_LENGTH}
506	+ * - {@link Piwik\Metrics::INDEX_BOUNCE_COUNT}
507	+ * - {@link Piwik\Metrics::INDEX_NB_VISITS_CONVERTED}
508	+ * @param bool\|\Piwik\RankingQuery $rankingQuery
509	+ * A pre-configured ranking query instance that will be used to limit the result.
510	+ * If set, the return value is the array returned by {@link Piwik\RankingQuery::execute()}.
511	+ *
512	+ * @return mixed A Zend_Db_Statement if `$rankingQuery` isn't supplied, otherwise the result of
513	+ * {@link Piwik\RankingQuery::execute()}. Read {@link queryVisitsByDimension() this}
514	+ * to see what aggregate data is calculated by the query.
515	+ * @api
516	+ */
517	+ public function queryVisitsByDimension(array $dimensions = array(), $where = false, array $additionalSelects = array(),
518	+ $metrics = false, $rankingQuery = false, $orderBy = false)
519	+ {
520	+ $tableName = self::LOG_VISIT_TABLE;
521	+ $availableMetrics = $this->getVisitsMetricFields();
522	+
523	+ $select = $this->getSelectStatement($dimensions, $tableName, $additionalSelects, $availableMetrics, $metrics);
524	+ $from = array($tableName);
525	+ $where = $this->getWhereStatement($tableName, self::VISIT_DATETIME_FIELD, $where);
526	+ $groupBy = $this->getGroupByStatement($dimensions, $tableName);
527	+ $orderBys = $orderBy ? [$orderBy] : [];
528	+
529	+ if ($rankingQuery) {
530	+ $orderBys[] = '`' . Metrics::INDEX_NB_VISITS . '` DESC';
531	+ }
532	+
533	+ $query = $this->generateQuery($select, $from, $where, $groupBy, implode(', ', $orderBys));
534	+
535	+ if ($rankingQuery) {
536	+ unset($availableMetrics[Metrics::INDEX_MAX_ACTIONS]);
537	+
538	+ // INDEX_NB_UNIQ_FINGERPRINTS is only processed if specifically asked for
539	+ if (!$this->isMetricRequested(Metrics::INDEX_NB_UNIQ_FINGERPRINTS, $metrics)) {
540	+ unset($availableMetrics[Metrics::INDEX_NB_UNIQ_FINGERPRINTS]);
541	+ }
542	+
543	+ $sumColumns = array_keys($availableMetrics);
544	+
545	+ if ($metrics) {
546	+ $sumColumns = array_intersect($sumColumns, $metrics);
547	+ }
548	+
549	+ $rankingQuery->addColumn($sumColumns, 'sum');
550	+ if ($this->isMetricRequested(Metrics::INDEX_MAX_ACTIONS, $metrics)) {
551	+ $rankingQuery->addColumn(Metrics::INDEX_MAX_ACTIONS, 'max');
552	+ }
553	+
554	+ return $rankingQuery->execute($query['sql'], $query['bind']);
555	+ }
556	+
557	+ return $this->getDb()->query($query['sql'], $query['bind']);
558	+ }
559	+
560	+ protected function getSelectsMetrics($metricsAvailable, $metricsRequested = false)
561	+ {
562	+ $selects = array();
563	+
564	+ foreach ($metricsAvailable as $metricId => $statement) {
565	+ if ($this->isMetricRequested($metricId, $metricsRequested)) {
566	+ $aliasAs = $this->getSelectAliasAs($metricId);
567	+ $selects[] = $statement . $aliasAs;
568	+ }
569	+ }
570	+
571	+ return $selects;
572	+ }
573	+
574	+ protected function getSelectStatement($dimensions, $tableName, $additionalSelects, array $availableMetrics, $requestedMetrics = false)
575	+ {
576	+ $dimensionsToSelect = $this->getDimensionsToSelect($dimensions, $additionalSelects);
577	+
578	+ $selects = array_merge(
579	+ $this->getSelectDimensions($dimensionsToSelect, $tableName),
580	+ $this->getSelectsMetrics($availableMetrics, $requestedMetrics),
581	+ !empty($additionalSelects) ? $additionalSelects : array()
582	+ );
583	+
584	+ $select = implode(self::FIELDS_SEPARATOR, $selects);
585	+ return $select;
586	+ }
587	+
588	+ /**
589	+ * Will return the subset of $dimensions that are not found in $additionalSelects
590	+ *
591	+ * @param $dimensions
592	+ * @param array $additionalSelects
593	+ * @return array
594	+ */
595	+ protected function getDimensionsToSelect($dimensions, $additionalSelects)
596	+ {
597	+ if (empty($additionalSelects)) {
598	+ return $dimensions;
599	+ }
600	+
601	+ $dimensionsToSelect = array();
602	+ foreach ($dimensions as $selectAs => $dimension) {
603	+ $asAlias = $this->getSelectAliasAs($dimension);
604	+ foreach ($additionalSelects as $additionalSelect) {
605	+ if (strpos($additionalSelect, $asAlias) === false) {
606	+ $dimensionsToSelect[$selectAs] = $dimension;
607	+ }
608	+ }
609	+ }
610	+
611	+ $dimensionsToSelect = array_unique($dimensionsToSelect);
612	+ return $dimensionsToSelect;
613	+ }
614	+
615	+ /**
616	+ * Returns the dimensions array, where
617	+ * (1) the table name is prepended to the field
618	+ * (2) the "AS `label` " is appended to the field
619	+ *
620	+ * @param $dimensions
621	+ * @param $tableName
622	+ * @param bool $appendSelectAs
623	+ * @param bool $parseSelectAs
624	+ * @return mixed
625	+ */
626	+ protected function getSelectDimensions($dimensions, $tableName, $appendSelectAs = true)
627	+ {
628	+ foreach ($dimensions as $selectAs => &$field) {
629	+ $selectAsString = $field;
630	+
631	+ if (!is_numeric($selectAs)) {
632	+ $selectAsString = $selectAs;
633	+ } else if ($this->isFieldFunctionOrComplexExpression($field)) {
634	+ // if complex expression has a select as, use it
635	+ if (!$appendSelectAs && preg_match('/\s+AS\s+(.?)\s$/', $field, $matches)) {
636	+ $field = $matches[1];
637	+ continue;
638	+ }
639	+
640	+ // if function w/o select as, do not alias or prefix
641	+ $selectAsString = $appendSelectAs = false;
642	+ }
643	+
644	+ $isKnownField = !in_array($field, array('referrer_data'));
645	+
646	+ if ($selectAsString == $field && $isKnownField) {
647	+ $field = $this->prefixColumn($field, $tableName);
648	+ }
649	+
650	+ if ($appendSelectAs && $selectAsString) {
651	+ $field = $this->prefixColumn($field, $tableName) . $this->getSelectAliasAs($selectAsString);
652	+ }
653	+ }
654	+
655	+ return $dimensions;
656	+ }
657	+
658	+ /**
659	+ * Prefixes a column name with a table name if not already done.
660	+ *
661	+ * @param string $column eg, 'location_provider'
662	+ * @param string $tableName eg, 'log_visit'
663	+ * @return string eg, 'log_visit.location_provider'
664	+ */
665	+ private function prefixColumn($column, $tableName)
666	+ {
667	+ if (strpos($column, '.') === false) {
668	+ return $tableName . '.' . $column;
669	+ } else {
670	+ return $column;
671	+ }
672	+ }
673	+
674	+ protected function isFieldFunctionOrComplexExpression($field)
675	+ {
676	+ return strpos($field, "(") !== false
677	+ \|\| strpos($field, "CASE") !== false;
678	+ }
679	+
680	+ protected function getSelectAliasAs($metricId)
681	+ {
682	+ return " AS `" . $metricId . "`";
683	+ }
684	+
685	+ protected function isMetricRequested($metricId, $metricsRequested)
686	+ {
687	+ // do not process INDEX_NB_UNIQ_FINGERPRINTS unless specifically asked for
688	+ if ($metricsRequested === false) {
689	+ if ($metricId == Metrics::INDEX_NB_UNIQ_FINGERPRINTS) {
690	+ return false;
691	+ }
692	+ return true;
693	+ }
694	+ return in_array($metricId, $metricsRequested);
695	+ }
696	+
697	+ public function getWhereStatement($tableName, $datetimeField, $extraWhere = false)
698	+ {
699	+ $where = "$tableName.$datetimeField >= ?
700	+ AND $tableName.$datetimeField <= ?
701	+ AND $tableName.idsite IN (". Common::getSqlStringFieldsArray($this->sites) . ")";
702	+
703	+ if (!empty($extraWhere)) {
704	+ $extraWhere = sprintf($extraWhere, $tableName, $tableName);
705	+ $where .= ' AND ' . $extraWhere;
706	+ }
707	+
708	+ return $where;
709	+ }
710	+
711	+ protected function getGroupByStatement($dimensions, $tableName)
712	+ {
713	+ $dimensions = $this->getSelectDimensions($dimensions, $tableName, $appendSelectAs = false);
714	+ $groupBy = implode(", ", $dimensions);
715	+
716	+ return $groupBy;
717	+ }
718	+
719	+ /**
720	+ * Returns general bind parameters for all log aggregation queries. This includes the datetime
721	+ * start of entities, datetime end of entities and IDs of all sites.
722	+ *
723	+ * @return array
724	+ */
725	+ public function getGeneralQueryBindParams()
726	+ {
727	+ $bind = array($this->dateStart->toString(Date::DATE_TIME_FORMAT), $this->dateEnd->toString(Date::DATE_TIME_FORMAT));
728	+ $bind = array_merge($bind, $this->sites);
729	+
730	+ return $bind;
731	+ }
732	+
733	+ /**
734	+ * Executes and returns a query aggregating ecommerce item data (everything stored in the
735	+ * log\_conversion\_item table) and returns a DB statement that can be used to iterate over the result
736	+ *
737	+ * <a name="queryEcommerceItems-result-set"></a>
738	+ * Result Set
739	+ *
740	+ * Each row of the result set represents an aggregated group of ecommerce items. The following
741	+ * columns are in each row of the result set:
742	+ *
743	+ * - {@link Piwik\Metrics::INDEX_ECOMMERCE_ITEM_REVENUE}: The total revenue for the group of items.
744	+ * - {@link Piwik\Metrics::INDEX_ECOMMERCE_ITEM_QUANTITY}: The total number of items in this group.
745	+ * - {@link Piwik\Metrics::INDEX_ECOMMERCE_ITEM_PRICE}: The total price for the group of items.
746	+ * - {@link Piwik\Metrics::INDEX_ECOMMERCE_ORDERS}: The total number of orders this group of items
747	+ * belongs to. This will be <= to the total number
748	+ * of items in this group.
749	+ * - {@link Piwik\Metrics::INDEX_NB_VISITS}: The total number of visits that caused these items to be logged.
750	+ * - ecommerceType: Either {@link Piwik\Tracker\GoalManager::IDGOAL_CART} if the items in this group were
751	+ * abandoned by a visitor, or {@link Piwik\Tracker\GoalManager::IDGOAL_ORDER} if they
752	+ * were ordered by a visitor.
753	+ *
754	+ * Limitations
755	+ *
756	+ * Segmentation is not yet supported for this aggregation method.
757	+ *
758	+ * @param string $dimension One or more log\_conversion\_item columns to group aggregated data by.
759	+ * Eg, `'idaction_sku'` or `'idaction_sku, idaction_category'`.
760	+ * @return \Zend_Db_Statement A statement object that can be used to iterate through the query's
761	+ * result set. See [above](#queryEcommerceItems-result-set) to learn more
762	+ * about what this query selects.
763	+ * @api
764	+ */
765	+ public function queryEcommerceItems($dimension)
766	+ {
767	+ $query = $this->generateQuery(
768	+ // SELECT ...
769	+ implode(
770	+ ', ',
771	+ array(
772	+ "log_action.name AS label",
773	+ sprintf("log_conversion_item.%s AS labelIdAction", $dimension),
774	+ sprintf(
775	+ '%s AS `%d`',
776	+ self::getSqlRevenue('SUM(log_conversion_item.quantity * log_conversion_item.price)'),
777	+ Metrics::INDEX_ECOMMERCE_ITEM_REVENUE
778	+ ),
779	+ sprintf(
780	+ '%s AS `%d`',
781	+ self::getSqlRevenue('SUM(log_conversion_item.quantity)'),
782	+ Metrics::INDEX_ECOMMERCE_ITEM_QUANTITY
783	+ ),
784	+ sprintf(
785	+ '%s AS `%d`',
786	+ self::getSqlRevenue('SUM(log_conversion_item.price)'),
787	+ Metrics::INDEX_ECOMMERCE_ITEM_PRICE
788	+ ),
789	+ sprintf(
790	+ 'COUNT(distinct log_conversion_item.idorder) AS `%d`',
791	+ Metrics::INDEX_ECOMMERCE_ORDERS
792	+ ),
793	+ sprintf(
794	+ 'COUNT(distinct log_conversion_item.idvisit) AS `%d`',
795	+ Metrics::INDEX_NB_VISITS
796	+ ),
797	+ sprintf(
798	+ 'CASE log_conversion_item.idorder WHEN \'0\' THEN %d ELSE %d END AS ecommerceType',
799	+ GoalManager::IDGOAL_CART,
800	+ GoalManager::IDGOAL_ORDER
801	+ )
802	+ )
803	+ ),
804	+
805	+ // FROM ...
806	+ array(
807	+ "log_conversion_item",
808	+ array(
809	+ "table" => "log_action",
810	+ "joinOn" => sprintf("log_conversion_item.%s = log_action.idaction", $dimension)
811	+ )
812	+ ),
813	+
814	+ // WHERE ... AND ...
815	+ implode(
816	+ ' AND ',
817	+ array(
818	+ 'log_conversion_item.server_time >= ?',
819	+ 'log_conversion_item.server_time <= ?',
820	+ 'log_conversion_item.idsite IN (' . Common::getSqlStringFieldsArray($this->sites) . ')',
821	+ 'log_conversion_item.deleted = 0'
822	+ )
823	+ ),
824	+
825	+ // GROUP BY ...
826	+ sprintf(
827	+ "ecommerceType, log_conversion_item.%s",
828	+ $dimension
829	+ ),
830	+
831	+ // ORDER ...
832	+ false
833	+ );
834	+
835	+ return $this->getDb()->query($query['sql'], $query['bind']);
836	+ }
837	+
838	+ /**
839	+ * Executes and returns a query aggregating action data (everything in the log_action table) and returns
840	+ * a DB statement that can be used to iterate over the result
841	+ *
842	+ * <a name="queryActionsByDimension-result-set"></a>
843	+ * Result Set
844	+ *
845	+ * Each row of the result set represents an aggregated group of actions. The following columns
846	+ * are in each aggregate row:
847	+ *
848	+ * - {@link Piwik\Metrics::INDEX_NB_UNIQ_VISITORS}: The total number of unique visitors that performed
849	+ * the actions in this group.
850	+ * - {@link Piwik\Metrics::INDEX_NB_VISITS}: The total number of visits these actions belong to.
851	+ * - {@link Piwik\Metrics::INDEX_NB_ACTIONS}: The total number of actions in this aggregate group.
852	+ *
853	+ * Additional data can be selected through the `$additionalSelects` parameter.
854	+ *
855	+ * _Note: The metrics calculated by this query can be customized by the `$metrics` parameter._
856	+ *
857	+ * @param array\|string $dimensions One or more SELECT fields that will be used to group the log_action
858	+ * rows by. This parameter determines which log_action rows will be
859	+ * aggregated together.
860	+ * @param bool\|string $where Additional condition for the WHERE clause. Can be used to filter
861	+ * the set of visits that are considered for aggregation.
862	+ * @param array $additionalSelects Additional SELECT fields that are not included in the group by
863	+ * clause. These can be aggregate expressions, eg, `SUM(somecol)`.
864	+ * @param bool\|array $metrics The set of metrics to calculate and return. If `false`, the query will select
865	+ * all of them. The following values can be used:
866	+ *
867	+ * - {@link Piwik\Metrics::INDEX_NB_UNIQ_VISITORS}
868	+ * - {@link Piwik\Metrics::INDEX_NB_VISITS}
869	+ * - {@link Piwik\Metrics::INDEX_NB_ACTIONS}
870	+ * @param bool\|\Piwik\RankingQuery $rankingQuery
871	+ * A pre-configured ranking query instance that will be used to limit the result.
872	+ * If set, the return value is the array returned by {@link Piwik\RankingQuery::execute()}.
873	+ * @param bool\|string $joinLogActionOnColumn One or more columns from the log_link_visit_action table that
874	+ * log_action should be joined on. The table alias used for each join
875	+ * is `"log_action$i"` where `$i` is the index of the column in this
876	+ * array.
877	+ *
878	+ * If a string is used for this parameter, the table alias is not
879	+ * suffixed (since there is only one column).
880	+ * @param string $secondaryOrderBy A secondary order by clause for the ranking query
881	+ * @return mixed A Zend_Db_Statement if `$rankingQuery` isn't supplied, otherwise the result of
882	+ * {@link Piwik\RankingQuery::execute()}. Read [this](#queryEcommerceItems-result-set)
883	+ * to see what aggregate data is calculated by the query.
884	+ * @api
885	+ */
886	+ public function queryActionsByDimension(
887	+ $dimensions,
888	+ $where = '',
889	+ $additionalSelects = array(),
890	+ $metrics = false,
891	+ $rankingQuery = null,
892	+ $joinLogActionOnColumn = false,
893	+ $secondaryOrderBy = null
894	+ ) {
895	+ $tableName = self::LOG_ACTIONS_TABLE;
896	+ $availableMetrics = $this->getActionsMetricFields();
897	+
898	+ $select = $this->getSelectStatement($dimensions, $tableName, $additionalSelects, $availableMetrics, $metrics);
899	+ $from = array($tableName);
900	+ $where = $this->getWhereStatement($tableName, self::ACTION_DATETIME_FIELD, $where);
901	+ $groupBy = $this->getGroupByStatement($dimensions, $tableName);
902	+
903	+ if ($joinLogActionOnColumn !== false) {
904	+ $multiJoin = is_array($joinLogActionOnColumn);
905	+ if (!$multiJoin) {
906	+ $joinLogActionOnColumn = array($joinLogActionOnColumn);
907	+ }
908	+
909	+ foreach ($joinLogActionOnColumn as $i => $joinColumn) {
910	+ $tableAlias = 'log_action' . ($multiJoin ? $i + 1 : '');
911	+
912	+ if (strpos($joinColumn, ' ') === false) {
913	+ $joinOn = $tableAlias . '.idaction = ' . $tableName . '.' . $joinColumn;
914	+ } else {
915	+ // more complex join column like if (...)
916	+ $joinOn = $tableAlias . '.idaction = ' . $joinColumn;
917	+ }
918	+
919	+ $from[] = array(
920	+ 'table' => 'log_action',
921	+ 'tableAlias' => $tableAlias,
922	+ 'joinOn' => $joinOn
923	+ );
924	+ }
925	+ }
926	+
927	+ $orderBy = false;
928	+ if ($rankingQuery) {
929	+ $orderBy = '`' . Metrics::INDEX_NB_ACTIONS . '` DESC';
930	+ if ($secondaryOrderBy) {
931	+ $orderBy .= ', ' . $secondaryOrderBy;
932	+ }
933	+ }
934	+
935	+ $query = $this->generateQuery($select, $from, $where, $groupBy, $orderBy);
936	+
937	+ if ($rankingQuery !== null) {
938	+ $sumColumns = array_keys($availableMetrics);
939	+ if ($metrics) {
940	+ $sumColumns = array_intersect($sumColumns, $metrics);
941	+ }
942	+
943	+ $rankingQuery->addColumn($sumColumns, 'sum');
944	+
945	+ return $rankingQuery->execute($query['sql'], $query['bind']);
946	+ }
947	+
948	+ return $this->getDb()->query($query['sql'], $query['bind']);
949	+ }
950	+
951	+ protected function getActionsMetricFields()
952	+ {
953	+ return array(
954	+ Metrics::INDEX_NB_VISITS => "count(distinct " . self::LOG_ACTIONS_TABLE . ".idvisit)",
955	+ Metrics::INDEX_NB_UNIQ_VISITORS => "count(distinct " . self::LOG_ACTIONS_TABLE . ".idvisitor)",
956	+ Metrics::INDEX_NB_ACTIONS => "count(*)",
957	+ );
958	+ }
959	+
960	+ /**
961	+ * Executes a query aggregating conversion data (everything in the log_conversion table) and returns
962	+ * a DB statement that can be used to iterate over the result.
963	+ *
964	+ * <a name="queryConversionsByDimension-result-set"></a>
965	+ * Result Set
966	+ *
967	+ * Each row of the result set represents an aggregated group of conversions. The
968	+ * following columns are in each aggregate row:
969	+ *
970	+ * - {@link Piwik\Metrics::INDEX_GOAL_NB_CONVERSIONS}: The total number of conversions in this aggregate
971	+ * group.
972	+ * - {@link Piwik\Metrics::INDEX_GOAL_NB_VISITS_CONVERTED}: The total number of visits during which these
973	+ * conversions were converted.
974	+ * - {@link Piwik\Metrics::INDEX_GOAL_REVENUE}: The total revenue generated by these conversions. This value
975	+ * includes the revenue from individual ecommerce items.
976	+ * - {@link Piwik\Metrics::INDEX_GOAL_ECOMMERCE_REVENUE_SUBTOTAL}: The total cost of all ecommerce items sold
977	+ * within these conversions. This value does not
978	+ * include tax, shipping or any applied discount.
979	+ *
980	+ * _This metric is only applicable to the special
981	+ * ecommerce goal (where `idGoal == 'ecommerceOrder'`)._
982	+ * - {@link Piwik\Metrics::INDEX_GOAL_ECOMMERCE_REVENUE_TAX}: The total tax applied to every transaction in these
983	+ * conversions.
984	+ *
985	+ * _This metric is only applicable to the special
986	+ * ecommerce goal (where `idGoal == 'ecommerceOrder'`)._
987	+ * - {@link Piwik\Metrics::INDEX_GOAL_ECOMMERCE_REVENUE_SHIPPING}: The total shipping cost for every transaction
988	+ * in these conversions.
989	+ *
990	+ * _This metric is only applicable to the special
991	+ * ecommerce goal (where `idGoal == 'ecommerceOrder'`)._
992	+ * - {@link Piwik\Metrics::INDEX_GOAL_ECOMMERCE_REVENUE_DISCOUNT}: The total discount applied to every transaction
993	+ * in these conversions.
994	+ *
995	+ * _This metric is only applicable to the special
996	+ * ecommerce goal (where `idGoal == 'ecommerceOrder'`)._
997	+ * - {@link Piwik\Metrics::INDEX_GOAL_ECOMMERCE_ITEMS}: The total number of ecommerce items sold in each transaction
998	+ * in these conversions.
999	+ *
1000	+ * _This metric is only applicable to the special
1001	+ * ecommerce goal (where `idGoal == 'ecommerceOrder'`)._
1002	+ *
1003	+ * Additional data can be selected through the `$additionalSelects` parameter.
1004	+ *
1005	+ * _Note: This method will only query the log_conversion table. Other tables cannot be joined
1006	+ * using this method._
1007	+ *
1008	+ * @param array\|string $dimensions One or more SELECT fields that will be used to group the log_conversion
1009	+ * rows by. This parameter determines which log_conversion rows will be
1010	+ * aggregated together.
1011	+ * @param bool\|string $where An optional SQL expression used in the SQL's WHERE clause.
1012	+ * @param array $additionalSelects Additional SELECT fields that are not included in the group by
1013	+ * clause. These can be aggregate expressions, eg, `SUM(somecol)`.
1014	+ * @return \Zend_Db_Statement
1015	+ */
1016	+ public function queryConversionsByDimension($dimensions = array(), $where = false, $additionalSelects = array(), $extraFrom

Matomo Analytics – Ethical Stats. Powerful Insights. - Version 1.0.2

Version Description

Release Info

Version 1.0.2