Line data Source code
1 : //* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 : /* This Source Code Form is subject to the terms of the Mozilla Public
3 : * License, v. 2.0. If a copy of the MPL was not distributed with this
4 : * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
5 :
6 : #include "Classifier.h"
7 : #include "LookupCacheV4.h"
8 : #include "nsIPrefBranch.h"
9 : #include "nsIPrefService.h"
10 : #include "nsISimpleEnumerator.h"
11 : #include "nsIRandomGenerator.h"
12 : #include "nsIInputStream.h"
13 : #include "nsISeekableStream.h"
14 : #include "nsIFile.h"
15 : #include "nsNetCID.h"
16 : #include "nsPrintfCString.h"
17 : #include "nsThreadUtils.h"
18 : #include "mozilla/EndianUtils.h"
19 : #include "mozilla/Telemetry.h"
20 : #include "mozilla/IntegerPrintfMacros.h"
21 : #include "mozilla/Logging.h"
22 : #include "mozilla/SyncRunnable.h"
23 : #include "mozilla/Base64.h"
24 : #include "mozilla/Unused.h"
25 : #include "mozilla/UniquePtr.h"
26 : #include "nsIUrlClassifierUtils.h"
27 : #include "nsUrlClassifierDBService.h"
28 :
29 : // MOZ_LOG=UrlClassifierDbService:5
30 : extern mozilla::LazyLogModule gUrlClassifierDbServiceLog;
31 : #define LOG(args) MOZ_LOG(gUrlClassifierDbServiceLog, mozilla::LogLevel::Debug, args)
32 : #define LOG_ENABLED() MOZ_LOG_TEST(gUrlClassifierDbServiceLog, mozilla::LogLevel::Debug)
33 :
34 : #define STORE_DIRECTORY NS_LITERAL_CSTRING("safebrowsing")
35 : #define TO_DELETE_DIR_SUFFIX NS_LITERAL_CSTRING("-to_delete")
36 : #define BACKUP_DIR_SUFFIX NS_LITERAL_CSTRING("-backup")
37 : #define UPDATING_DIR_SUFFIX NS_LITERAL_CSTRING("-updating")
38 :
39 : #define METADATA_SUFFIX NS_LITERAL_CSTRING(".metadata")
40 :
41 : namespace mozilla {
42 : namespace safebrowsing {
43 :
44 : void
45 5 : Classifier::SplitTables(const nsACString& str, nsTArray<nsCString>& tables)
46 : {
47 5 : tables.Clear();
48 :
49 15 : nsACString::const_iterator begin, iter, end;
50 5 : str.BeginReading(begin);
51 0 : str.EndReading(end);
52 0 : while (begin != end) {
53 69 : iter = begin;
54 0 : FindCharInReadable(',', iter, end);
55 0 : nsDependentCSubstring table = Substring(begin,iter);
56 69 : if (!table.IsEmpty()) {
57 0 : tables.AppendElement(Substring(begin, iter));
58 : }
59 0 : begin = iter;
60 0 : if (begin != end) {
61 0 : begin++;
62 : }
63 : }
64 5 : }
65 :
66 : nsresult
67 0 : Classifier::GetPrivateStoreDirectory(nsIFile* aRootStoreDirectory,
68 : const nsACString& aTableName,
69 : const nsACString& aProvider,
70 : nsIFile** aPrivateStoreDirectory)
71 : {
72 0 : NS_ENSURE_ARG_POINTER(aPrivateStoreDirectory);
73 :
74 0 : if (!StringEndsWith(aTableName, NS_LITERAL_CSTRING("-proto"))) {
75 : // Only V4 table names (ends with '-proto') would be stored
76 : // to per-provider sub-directory.
77 0 : nsCOMPtr<nsIFile>(aRootStoreDirectory).forget(aPrivateStoreDirectory);
78 0 : return NS_OK;
79 : }
80 :
81 0 : if (aProvider.IsEmpty()) {
82 : // When failing to get provider, just store in the root folder.
83 0 : nsCOMPtr<nsIFile>(aRootStoreDirectory).forget(aPrivateStoreDirectory);
84 0 : return NS_OK;
85 : }
86 :
87 0 : nsCOMPtr<nsIFile> providerDirectory;
88 :
89 : // Clone first since we are gonna create a new directory.
90 0 : nsresult rv = aRootStoreDirectory->Clone(getter_AddRefs(providerDirectory));
91 0 : NS_ENSURE_SUCCESS(rv, rv);
92 :
93 : // Append the provider name to the root store directory.
94 0 : rv = providerDirectory->AppendNative(aProvider);
95 0 : NS_ENSURE_SUCCESS(rv, rv);
96 :
97 : // Ensure existence of the provider directory.
98 : bool dirExists;
99 0 : rv = providerDirectory->Exists(&dirExists);
100 0 : NS_ENSURE_SUCCESS(rv, rv);
101 :
102 0 : if (!dirExists) {
103 0 : LOG(("Creating private directory for %s", nsCString(aTableName).get()));
104 0 : rv = providerDirectory->Create(nsIFile::DIRECTORY_TYPE, 0755);
105 0 : NS_ENSURE_SUCCESS(rv, rv);
106 0 : providerDirectory.forget(aPrivateStoreDirectory);
107 0 : return rv;
108 : }
109 :
110 : // Store directory exists. Check if it's a directory.
111 : bool isDir;
112 0 : rv = providerDirectory->IsDirectory(&isDir);
113 0 : NS_ENSURE_SUCCESS(rv, rv);
114 0 : if (!isDir) {
115 : return NS_ERROR_FILE_DESTINATION_NOT_DIR;
116 : }
117 :
118 0 : providerDirectory.forget(aPrivateStoreDirectory);
119 :
120 0 : return NS_OK;
121 : }
122 :
123 1 : Classifier::Classifier()
124 : : mIsTableRequestResultOutdated(true)
125 11 : , mUpdateInterrupted(true)
126 : {
127 0 : NS_NewNamedThread(NS_LITERAL_CSTRING("Classifier Update"),
128 5 : getter_AddRefs(mUpdateThread));
129 0 : }
130 :
131 0 : Classifier::~Classifier()
132 : {
133 0 : Close();
134 0 : }
135 :
136 : nsresult
137 1 : Classifier::SetupPathNames()
138 : {
139 : // Get the root directory where to store all the databases.
140 0 : nsresult rv = mCacheDirectory->Clone(getter_AddRefs(mRootStoreDirectory));
141 0 : NS_ENSURE_SUCCESS(rv, rv);
142 :
143 3 : rv = mRootStoreDirectory->AppendNative(STORE_DIRECTORY);
144 1 : NS_ENSURE_SUCCESS(rv, rv);
145 :
146 : // Make sure LookupCaches (which are persistent and survive updates)
147 : // are reading/writing in the right place. We will be moving their
148 : // files "underneath" them during backup/restore.
149 2 : for (uint32_t i = 0; i < mLookupCaches.Length(); i++) {
150 0 : mLookupCaches[i]->UpdateRootDirHandle(mRootStoreDirectory);
151 : }
152 :
153 : // Directory where to move a backup before an update.
154 0 : rv = mCacheDirectory->Clone(getter_AddRefs(mBackupDirectory));
155 0 : NS_ENSURE_SUCCESS(rv, rv);
156 :
157 5 : rv = mBackupDirectory->AppendNative(STORE_DIRECTORY + BACKUP_DIR_SUFFIX);
158 0 : NS_ENSURE_SUCCESS(rv, rv);
159 :
160 : // Directory where to be working on the update.
161 0 : rv = mCacheDirectory->Clone(getter_AddRefs(mUpdatingDirectory));
162 1 : NS_ENSURE_SUCCESS(rv, rv);
163 :
164 0 : rv = mUpdatingDirectory->AppendNative(STORE_DIRECTORY + UPDATING_DIR_SUFFIX);
165 1 : NS_ENSURE_SUCCESS(rv, rv);
166 :
167 : // Directory where to move the backup so we can atomically
168 : // delete (really move) it.
169 2 : rv = mCacheDirectory->Clone(getter_AddRefs(mToDeleteDirectory));
170 0 : NS_ENSURE_SUCCESS(rv, rv);
171 :
172 5 : rv = mToDeleteDirectory->AppendNative(STORE_DIRECTORY + TO_DELETE_DIR_SUFFIX);
173 1 : NS_ENSURE_SUCCESS(rv, rv);
174 :
175 : return NS_OK;
176 : }
177 :
178 : nsresult
179 1 : Classifier::CreateStoreDirectory()
180 : {
181 : // Ensure the safebrowsing directory exists.
182 : bool storeExists;
183 1 : nsresult rv = mRootStoreDirectory->Exists(&storeExists);
184 0 : NS_ENSURE_SUCCESS(rv, rv);
185 :
186 1 : if (!storeExists) {
187 1 : rv = mRootStoreDirectory->Create(nsIFile::DIRECTORY_TYPE, 0755);
188 0 : NS_ENSURE_SUCCESS(rv, rv);
189 : } else {
190 : bool storeIsDir;
191 0 : rv = mRootStoreDirectory->IsDirectory(&storeIsDir);
192 0 : NS_ENSURE_SUCCESS(rv, rv);
193 0 : if (!storeIsDir)
194 : return NS_ERROR_FILE_DESTINATION_NOT_DIR;
195 : }
196 :
197 : return NS_OK;
198 : }
199 :
200 : nsresult
201 1 : Classifier::Open(nsIFile& aCacheDirectory)
202 : {
203 : // Remember the Local profile directory.
204 2 : nsresult rv = aCacheDirectory.Clone(getter_AddRefs(mCacheDirectory));
205 1 : NS_ENSURE_SUCCESS(rv, rv);
206 :
207 : // Create the handles to the update and backup directories.
208 1 : rv = SetupPathNames();
209 1 : NS_ENSURE_SUCCESS(rv, rv);
210 :
211 : // Clean up any to-delete directories that haven't been deleted yet.
212 : // This is still required for backward compatibility.
213 0 : rv = CleanToDelete();
214 0 : NS_ENSURE_SUCCESS(rv, rv);
215 :
216 : // If we met a crash during the previous update, "safebrowsing-updating"
217 : // directory will exist and let's remove it.
218 0 : rv = mUpdatingDirectory->Remove(true);
219 0 : if (NS_SUCCEEDED(rv)) {
220 : // If the "safebrowsing-updating" exists, it implies a crash occurred
221 : // in the previous update.
222 0 : LOG(("We may have hit a crash in the previous update."));
223 : }
224 :
225 : // Check whether we have an incomplete update and recover from the
226 : // backup if so.
227 1 : rv = RecoverBackups();
228 0 : NS_ENSURE_SUCCESS(rv, rv);
229 :
230 : // Make sure the main store directory exists.
231 0 : rv = CreateStoreDirectory();
232 0 : NS_ENSURE_SUCCESS(rv, rv);
233 :
234 : // Build the list of know urlclassifier lists
235 : // XXX: Disk IO potentially on the main thread during startup
236 0 : RegenActiveTables();
237 :
238 1 : return NS_OK;
239 : }
240 :
241 : void
242 0 : Classifier::Close()
243 : {
244 : // Close will be called by PreShutdown, so it is important to note that
245 : // things put here should not affect an ongoing update thread.
246 0 : DropStores();
247 0 : }
248 :
249 : void
250 0 : Classifier::Reset()
251 : {
252 0 : MOZ_ASSERT(NS_GetCurrentThread() != mUpdateThread,
253 : "Reset() MUST NOT be called on update thread");
254 :
255 0 : LOG(("Reset() is called so we interrupt the update."));
256 0 : mUpdateInterrupted = true;
257 :
258 0 : auto resetFunc = [=] {
259 0 : DropStores();
260 :
261 0 : mRootStoreDirectory->Remove(true);
262 0 : mBackupDirectory->Remove(true);
263 0 : mUpdatingDirectory->Remove(true);
264 0 : mToDeleteDirectory->Remove(true);
265 :
266 0 : CreateStoreDirectory();
267 :
268 0 : RegenActiveTables();
269 0 : };
270 :
271 0 : if (!mUpdateThread) {
272 0 : LOG(("Async update has been disabled. Just Reset() on worker thread."));
273 0 : resetFunc();
274 0 : return;
275 : }
276 :
277 : nsCOMPtr<nsIRunnable> r =
278 0 : NS_NewRunnableFunction("safebrowsing::Classifier::Reset", resetFunc);
279 0 : SyncRunnable::DispatchToThread(mUpdateThread, r);
280 : }
281 :
282 : void
283 0 : Classifier::ResetTables(ClearType aType, const nsTArray<nsCString>& aTables)
284 : {
285 0 : for (uint32_t i = 0; i < aTables.Length(); i++) {
286 0 : LOG(("Resetting table: %s", aTables[i].get()));
287 0 : RefPtr<LookupCache> cache = GetLookupCache(aTables[i]);
288 0 : if (cache) {
289 : // Remove any cached Completes for this table if clear type is Clear_Cache
290 0 : if (aType == Clear_Cache) {
291 0 : cache->ClearCache();
292 : } else {
293 0 : cache->ClearAll();
294 : }
295 : }
296 : }
297 :
298 : // Clear on-disk database if clear type is Clear_All
299 0 : if (aType == Clear_All) {
300 0 : DeleteTables(mRootStoreDirectory, aTables);
301 :
302 0 : RegenActiveTables();
303 : }
304 0 : }
305 :
306 : void
307 0 : Classifier::DeleteTables(nsIFile* aDirectory, const nsTArray<nsCString>& aTables)
308 : {
309 0 : nsCOMPtr<nsIDirectoryEnumerator> entries;
310 0 : nsresult rv = aDirectory->GetDirectoryEntries(getter_AddRefs(entries));
311 0 : NS_ENSURE_SUCCESS_VOID(rv);
312 :
313 0 : nsCOMPtr<nsIFile> file;
314 0 : while (NS_SUCCEEDED(rv = entries->GetNextFile(getter_AddRefs(file))) && file) {
315 : // If |file| is a directory, recurse to find its entries as well.
316 : bool isDirectory;
317 0 : if (NS_FAILED(file->IsDirectory(&isDirectory))) {
318 0 : continue;
319 : }
320 0 : if (isDirectory) {
321 0 : DeleteTables(file, aTables);
322 0 : continue;
323 : }
324 :
325 0 : nsCString leafName;
326 0 : rv = file->GetNativeLeafName(leafName);
327 0 : NS_ENSURE_SUCCESS_VOID(rv);
328 :
329 : // Remove file extension if there's one.
330 0 : int32_t dotPosition = leafName.RFind(".");
331 0 : if (dotPosition >= 0) {
332 0 : leafName.Truncate(dotPosition);
333 : }
334 :
335 0 : if (!leafName.IsEmpty() && aTables.Contains(leafName)) {
336 0 : if (NS_FAILED(file->Remove(false))) {
337 0 : NS_WARNING(nsPrintfCString("Fail to remove file %s from the disk",
338 0 : leafName.get()).get());
339 : }
340 : }
341 : }
342 0 : NS_ENSURE_SUCCESS_VOID(rv);
343 : }
344 :
345 : void
346 0 : Classifier::TableRequest(nsACString& aResult)
347 : {
348 0 : MOZ_ASSERT(!NS_IsMainThread(),
349 : "TableRequest must be called on the classifier worker thread.");
350 :
351 : // This function and all disk I/O are guaranteed to occur
352 : // on the same thread so we don't need to add a lock around.
353 0 : if (!mIsTableRequestResultOutdated) {
354 0 : aResult = mTableRequestResult;
355 0 : return;
356 : }
357 :
358 : // Generating v2 table info.
359 0 : nsTArray<nsCString> tables;
360 0 : ActiveTables(tables);
361 0 : for (uint32_t i = 0; i < tables.Length(); i++) {
362 0 : HashStore store(tables[i], GetProvider(tables[i]), mRootStoreDirectory);
363 :
364 0 : nsresult rv = store.Open();
365 0 : if (NS_FAILED(rv)) {
366 0 : continue;
367 : }
368 :
369 0 : ChunkSet &adds = store.AddChunks();
370 0 : ChunkSet &subs = store.SubChunks();
371 :
372 : // Open HashStore will always succeed even that is not a v2 table.
373 : // So skip tables without add and sub chunks.
374 0 : if (adds.Length() == 0 && subs.Length() == 0) {
375 : continue;
376 : }
377 :
378 0 : aResult.Append(store.TableName());
379 0 : aResult.Append(';');
380 :
381 0 : if (adds.Length() > 0) {
382 0 : aResult.AppendLiteral("a:");
383 0 : nsAutoCString addList;
384 0 : adds.Serialize(addList);
385 0 : aResult.Append(addList);
386 : }
387 :
388 0 : if (subs.Length() > 0) {
389 0 : if (adds.Length() > 0)
390 0 : aResult.Append(':');
391 0 : aResult.AppendLiteral("s:");
392 0 : nsAutoCString subList;
393 0 : subs.Serialize(subList);
394 0 : aResult.Append(subList);
395 : }
396 :
397 0 : aResult.Append('\n');
398 : }
399 :
400 : // Load meta data from *.metadata files in the root directory.
401 : // Specifically for v4 tables.
402 0 : nsCString metadata;
403 0 : nsresult rv = LoadMetadata(mRootStoreDirectory, metadata);
404 0 : if (NS_SUCCEEDED(rv)) {
405 0 : aResult.Append(metadata);
406 : }
407 :
408 : // Update the TableRequest result in-memory cache.
409 0 : mTableRequestResult = aResult;
410 0 : mIsTableRequestResultOutdated = false;
411 : }
412 :
413 : nsresult
414 0 : Classifier::Check(const nsACString& aSpec,
415 : const nsACString& aTables,
416 : LookupResultArray& aResults)
417 : {
418 0 : Telemetry::AutoTimer<Telemetry::URLCLASSIFIER_CL_CHECK_TIME> timer;
419 :
420 : // Get the set of fragments based on the url. This is necessary because we
421 : // only look up at most 5 URLs per aSpec, even if aSpec has more than 5
422 : // components.
423 0 : nsTArray<nsCString> fragments;
424 0 : nsresult rv = LookupCache::GetLookupFragments(aSpec, &fragments);
425 0 : NS_ENSURE_SUCCESS(rv, rv);
426 :
427 0 : nsTArray<nsCString> activeTables;
428 0 : SplitTables(aTables, activeTables);
429 :
430 0 : LookupCacheArray cacheArray;
431 0 : for (uint32_t i = 0; i < activeTables.Length(); i++) {
432 0 : LOG(("Checking table %s", activeTables[i].get()));
433 0 : RefPtr<LookupCache> cache = GetLookupCache(activeTables[i]);
434 0 : if (cache) {
435 0 : cacheArray.AppendElement(cache);
436 : } else {
437 0 : return NS_ERROR_FAILURE;
438 : }
439 : }
440 :
441 : // Now check each lookup fragment against the entries in the DB.
442 0 : for (uint32_t i = 0; i < fragments.Length(); i++) {
443 : Completion lookupHash;
444 0 : lookupHash.FromPlaintext(fragments[i]);
445 :
446 0 : if (LOG_ENABLED()) {
447 0 : nsAutoCString checking;
448 0 : lookupHash.ToHexString(checking);
449 0 : LOG(("Checking fragment %s, hash %s (%X)", fragments[i].get(),
450 : checking.get(), lookupHash.ToUint32()));
451 : }
452 :
453 0 : for (uint32_t i = 0; i < cacheArray.Length(); i++) {
454 0 : RefPtr<LookupCache> cache = cacheArray[i];
455 : bool has, confirmed;
456 : uint32_t matchLength;
457 :
458 0 : rv = cache->Has(lookupHash, &has, &matchLength, &confirmed);
459 0 : NS_ENSURE_SUCCESS(rv, rv);
460 :
461 0 : if (has) {
462 0 : RefPtr<LookupResult> result = new LookupResult;
463 0 : aResults.AppendElement(result);
464 :
465 0 : LOG(("Found a result in %s: %s",
466 : cache->TableName().get(),
467 : confirmed ? "confirmed." : "Not confirmed."));
468 :
469 0 : result->hash.complete = lookupHash;
470 0 : result->mConfirmed = confirmed;
471 0 : result->mTableName.Assign(cache->TableName());
472 0 : result->mPartialHashLength = confirmed ? COMPLETE_SIZE : matchLength;
473 0 : result->mProtocolV2 = LookupCache::Cast<LookupCacheV2>(cache);
474 : }
475 : }
476 : }
477 :
478 : return NS_OK;
479 : }
480 :
481 : static nsresult
482 0 : SwapDirectoryContent(nsIFile* aDir1,
483 : nsIFile* aDir2,
484 : nsIFile* aParentDir,
485 : nsIFile* aTempDir)
486 : {
487 : // Pre-condition: |aDir1| and |aDir2| are directory and their parent
488 : // are both |aParentDir|.
489 : //
490 : // Post-condition: The locations where aDir1 and aDir2 point to will not
491 : // change but their contents will be exchanged. If we failed
492 : // to swap their content, everything will be rolled back.
493 :
494 0 : nsAutoCString tempDirName;
495 0 : aTempDir->GetNativeLeafName(tempDirName);
496 :
497 : nsresult rv;
498 :
499 0 : nsAutoCString dirName1, dirName2;
500 0 : aDir1->GetNativeLeafName(dirName1);
501 0 : aDir2->GetNativeLeafName(dirName2);
502 :
503 0 : LOG(("Swapping directories %s and %s...", dirName1.get(),
504 : dirName2.get()));
505 :
506 : // 1. Rename "dirName1" to "temp"
507 0 : rv = aDir1->RenameToNative(nullptr, tempDirName);
508 0 : if (NS_FAILED(rv)) {
509 0 : LOG(("Unable to rename %s to %s", dirName1.get(),
510 : tempDirName.get()));
511 : return rv; // Nothing to roll back.
512 : }
513 :
514 : // 1.1. Create a handle for temp directory. This is required since
515 : // |nsIFile.rename| will not change the location where the
516 : // object points to.
517 0 : nsCOMPtr<nsIFile> tempDirectory;
518 0 : rv = aParentDir->Clone(getter_AddRefs(tempDirectory));
519 0 : rv = tempDirectory->AppendNative(tempDirName);
520 :
521 : // 2. Rename "dirName2" to "dirName1".
522 0 : rv = aDir2->RenameToNative(nullptr, dirName1);
523 0 : if (NS_FAILED(rv)) {
524 0 : LOG(("Failed to rename %s to %s. Rename temp directory back to %s",
525 : dirName2.get(), dirName1.get(), dirName1.get()));
526 0 : nsresult rbrv = tempDirectory->RenameToNative(nullptr, dirName1);
527 0 : NS_ENSURE_SUCCESS(rbrv, rbrv);
528 : return rv;
529 : }
530 :
531 : // 3. Rename "temp" to "dirName2".
532 0 : rv = tempDirectory->RenameToNative(nullptr, dirName2);
533 0 : if (NS_FAILED(rv)) {
534 0 : LOG(("Failed to rename temp directory to %s. ", dirName2.get()));
535 : // We've done (1) renaming "dir1 to temp" and
536 : // (2) renaming "dir2 to dir1"
537 : // so the rollback is
538 : // (1) renaming "dir1 to dir2" and
539 : // (2) renaming "temp to dir1"
540 : nsresult rbrv; // rollback result
541 0 : rbrv = aDir1->RenameToNative(nullptr, dirName2);
542 0 : NS_ENSURE_SUCCESS(rbrv, rbrv);
543 0 : rbrv = tempDirectory->RenameToNative(nullptr, dirName1);
544 0 : NS_ENSURE_SUCCESS(rbrv, rbrv);
545 : return rv;
546 : }
547 :
548 : return rv;
549 : }
550 :
551 : void
552 0 : Classifier::RemoveUpdateIntermediaries()
553 : {
554 : // Remove old LookupCaches.
555 0 : mNewLookupCaches.Clear();
556 :
557 : // Remove the "old" directory. (despite its looking-new name)
558 0 : if (NS_FAILED(mUpdatingDirectory->Remove(true))) {
559 : // If the directory is locked from removal for some reason,
560 : // we will fail here and it doesn't matter until the next
561 : // update. (the next udpate will fail due to the removable
562 : // "safebrowsing-udpating" directory.)
563 0 : LOG(("Failed to remove updating directory."));
564 : }
565 0 : }
566 :
567 : void
568 0 : Classifier::CopyAndInvalidateFullHashCache()
569 : {
570 0 : MOZ_ASSERT(NS_GetCurrentThread() != mUpdateThread,
571 : "CopyAndInvalidateFullHashCache cannot be called on update thread "
572 : "since it mutates mLookupCaches which is only safe on "
573 : "worker thread.");
574 :
575 : // New lookup caches are built from disk, data likes cache which is
576 : // generated online won't exist. We have to manually copy cache from
577 : // old LookupCache to new LookupCache.
578 0 : for (auto& newCache: mNewLookupCaches) {
579 0 : for (auto& oldCache: mLookupCaches) {
580 0 : if (oldCache->TableName() == newCache->TableName()) {
581 0 : newCache->CopyFullHashCache(oldCache);
582 0 : break;
583 : }
584 : }
585 : }
586 :
587 : // Clear cache when update.
588 : // Invalidate cache entries in CopyAndInvalidateFullHashCache because only
589 : // at this point we will have cache data in LookupCache.
590 0 : for (auto& newCache: mNewLookupCaches) {
591 0 : newCache->InvalidateExpiredCacheEntries();
592 : }
593 0 : }
594 :
595 : void
596 0 : Classifier::MergeNewLookupCaches()
597 : {
598 0 : MOZ_ASSERT(NS_GetCurrentThread() != mUpdateThread,
599 : "MergeNewLookupCaches cannot be called on update thread "
600 : "since it mutates mLookupCaches which is only safe on "
601 : "worker thread.");
602 :
603 0 : for (auto& newCache: mNewLookupCaches) {
604 : // For each element in mNewLookCaches, it will be swapped with
605 : // - An old cache in mLookupCache with the same table name or
606 : // - nullptr (mLookupCache will be expaned) otherwise.
607 0 : size_t swapIndex = 0;
608 0 : for (; swapIndex < mLookupCaches.Length(); swapIndex++) {
609 0 : if (mLookupCaches[swapIndex]->TableName() == newCache->TableName()) {
610 : break;
611 : }
612 : }
613 0 : if (swapIndex == mLookupCaches.Length()) {
614 0 : mLookupCaches.AppendElement(nullptr);
615 : }
616 :
617 0 : Swap(mLookupCaches[swapIndex], newCache);
618 0 : mLookupCaches[swapIndex]->UpdateRootDirHandle(mRootStoreDirectory);
619 : }
620 :
621 : // At this point, mNewLookupCaches's length remains the same but
622 : // will contain either old cache (override) or nullptr (append).
623 0 : }
624 :
625 : nsresult
626 0 : Classifier::SwapInNewTablesAndCleanup()
627 : {
628 : nsresult rv;
629 :
630 : // Step 1. Swap in on-disk tables. The idea of using "safebrowsing-backup"
631 : // as the intermediary directory is we can get databases recovered if
632 : // crash occurred in any step of the swap. (We will recover from
633 : // "safebrowsing-backup" in OpenDb().)
634 0 : rv = SwapDirectoryContent(mUpdatingDirectory, // contains new tables
635 : mRootStoreDirectory, // contains old tables
636 : mCacheDirectory, // common parent dir
637 0 : mBackupDirectory); // intermediary dir for swap
638 0 : if (NS_FAILED(rv)) {
639 0 : LOG(("Failed to swap in on-disk tables."));
640 0 : RemoveUpdateIntermediaries();
641 0 : return rv;
642 : }
643 :
644 : // Step 2. Merge mNewLookupCaches into mLookupCaches. The outdated
645 : // LookupCaches will be stored in mNewLookupCaches and be cleaned
646 : // up later.
647 0 : MergeNewLookupCaches();
648 :
649 : // Step 3. Re-generate active tables based on on-disk tables.
650 0 : rv = RegenActiveTables();
651 0 : if (NS_FAILED(rv)) {
652 0 : LOG(("Failed to re-generate active tables!"));
653 : }
654 :
655 : // Step 4. Clean up intermediaries for update.
656 0 : RemoveUpdateIntermediaries();
657 :
658 : // Step 5. Invalidate cached tableRequest request.
659 0 : mIsTableRequestResultOutdated = true;
660 :
661 0 : LOG(("Done swap in updated tables."));
662 :
663 : return rv;
664 : }
665 :
666 0 : void Classifier::FlushAndDisableAsyncUpdate()
667 : {
668 0 : LOG(("Classifier::FlushAndDisableAsyncUpdate [%p, %p]", this, mUpdateThread.get()));
669 :
670 0 : if (!mUpdateThread) {
671 0 : LOG(("Async update has been disabled."));
672 : return;
673 : }
674 :
675 0 : mUpdateThread->Shutdown();
676 0 : mUpdateThread = nullptr;
677 : }
678 :
679 : nsresult
680 0 : Classifier::AsyncApplyUpdates(const TableUpdateArray& aUpdates,
681 : const AsyncUpdateCallback& aCallback)
682 : {
683 0 : LOG(("Classifier::AsyncApplyUpdates"));
684 :
685 0 : if (!mUpdateThread) {
686 0 : LOG(("Async update has already been disabled."));
687 : return NS_ERROR_FAILURE;
688 : }
689 :
690 : // Caller thread | Update thread
691 : // --------------------------------------------------------
692 : // | ApplyUpdatesBackground
693 : // (processing other task) | (bg-update done. ping back to caller thread)
694 : // (processing other task) | idle...
695 : // ApplyUpdatesForeground |
696 : // callback |
697 :
698 0 : MOZ_ASSERT(mNewLookupCaches.IsEmpty(),
699 : "There should be no leftovers from a previous update.");
700 :
701 0 : mUpdateInterrupted = false;
702 0 : nsresult rv = mRootStoreDirectory->Clone(getter_AddRefs(mRootStoreDirectoryForUpdate));
703 0 : if (NS_FAILED(rv)) {
704 0 : LOG(("Failed to clone mRootStoreDirectory for update."));
705 : return rv;
706 : }
707 :
708 0 : nsCOMPtr<nsIThread> callerThread = NS_GetCurrentThread();
709 0 : MOZ_ASSERT(callerThread != mUpdateThread);
710 :
711 : nsCOMPtr<nsIRunnable> bgRunnable =
712 0 : NS_NewRunnableFunction("safebrowsing::Classifier::AsyncApplyUpdates", [=] {
713 0 : MOZ_ASSERT(NS_GetCurrentThread() == mUpdateThread,
714 : "MUST be on update thread");
715 :
716 : nsresult bgRv;
717 0 : nsCString failedTableName;
718 0 : TableUpdateArray updates;
719 :
720 : // Make a copy of the array since we'll be removing entries as
721 : // we process them on the background thread.
722 0 : if (updates.AppendElements(aUpdates, fallible)) {
723 0 : LOG(("Step 1. ApplyUpdatesBackground on update thread."));
724 0 : bgRv = ApplyUpdatesBackground(updates, failedTableName);
725 : } else {
726 0 : LOG(("Step 1. Not enough memory to run ApplyUpdatesBackground on update thread."));
727 : bgRv = NS_ERROR_OUT_OF_MEMORY;
728 : }
729 :
730 0 : nsCOMPtr<nsIRunnable> fgRunnable = NS_NewRunnableFunction(
731 0 : "safebrowsing::Classifier::AsyncApplyUpdates", [=] {
732 0 : MOZ_ASSERT(NS_GetCurrentThread() == callerThread,
733 : "MUST be on caller thread");
734 :
735 0 : LOG(("Step 2. ApplyUpdatesForeground on caller thread"));
736 0 : nsresult rv = ApplyUpdatesForeground(bgRv, failedTableName);
737 : ;
738 :
739 0 : LOG(("Step 3. Updates applied! Fire callback."));
740 :
741 0 : aCallback(rv);
742 0 : });
743 0 : callerThread->Dispatch(fgRunnable, NS_DISPATCH_NORMAL);
744 0 : });
745 :
746 0 : return mUpdateThread->Dispatch(bgRunnable, NS_DISPATCH_NORMAL);
747 : }
748 :
749 : nsresult
750 0 : Classifier::ApplyUpdatesBackground(TableUpdateArray& aUpdates,
751 : nsACString& aFailedTableName)
752 : {
753 : // |mUpdateInterrupted| is guaranteed to have been unset.
754 : // If |mUpdateInterrupted| is set at any point, Reset() must have
755 : // been called then we need to interrupt the update process.
756 : // We only add checkpoints for non-trivial tasks.
757 :
758 0 : if (aUpdates.IsEmpty()) {
759 : return NS_OK;
760 : }
761 :
762 : nsCOMPtr<nsIUrlClassifierUtils> urlUtil =
763 0 : do_GetService(NS_URLCLASSIFIERUTILS_CONTRACTID);
764 :
765 0 : nsCString provider;
766 : // Assume all TableUpdate objects should have the same provider.
767 0 : urlUtil->GetTelemetryProvider(aUpdates[0]->TableName(), provider);
768 :
769 : Telemetry::AutoTimer<Telemetry::URLCLASSIFIER_CL_KEYED_UPDATE_TIME>
770 0 : keyedTimer(provider);
771 :
772 0 : PRIntervalTime clockStart = 0;
773 0 : if (LOG_ENABLED()) {
774 0 : clockStart = PR_IntervalNow();
775 : }
776 :
777 : nsresult rv;
778 :
779 : // Check point 1: Copying file takes time so we check here.
780 0 : if (mUpdateInterrupted) {
781 0 : LOG(("Update is interrupted. Don't copy files."));
782 : return NS_OK;
783 : }
784 :
785 0 : rv = CopyInUseDirForUpdate(); // i.e. mUpdatingDirectory will be setup.
786 0 : if (NS_FAILED(rv)) {
787 0 : LOG(("Failed to copy in-use directory for update."));
788 : return rv;
789 : }
790 :
791 0 : LOG(("Applying %zu table updates.", aUpdates.Length()));
792 :
793 0 : for (uint32_t i = 0; i < aUpdates.Length(); i++) {
794 0 : RefPtr<const TableUpdate> update = aUpdates[i];
795 0 : if (!update) {
796 : // Previous UpdateHashStore() may have consumed this update..
797 0 : continue;
798 : }
799 :
800 : // Run all updates for one table
801 0 : nsAutoCString updateTable(update->TableName());
802 :
803 : // Check point 2: Processing downloaded data takes time.
804 0 : if (mUpdateInterrupted) {
805 0 : LOG(("Update is interrupted. Stop building new tables."));
806 0 : return NS_OK;
807 : }
808 :
809 : // Will update the mirrored in-memory and on-disk databases.
810 0 : if (TableUpdate::Cast<TableUpdateV2>(update)) {
811 0 : rv = UpdateHashStore(aUpdates, updateTable);
812 : } else {
813 0 : rv = UpdateTableV4(aUpdates, updateTable);
814 : }
815 :
816 0 : if (NS_FAILED(rv)) {
817 0 : aFailedTableName = updateTable;
818 0 : RemoveUpdateIntermediaries();
819 0 : return rv;
820 : }
821 : }
822 :
823 0 : if (LOG_ENABLED()) {
824 0 : PRIntervalTime clockEnd = PR_IntervalNow();
825 0 : LOG(("update took %dms\n",
826 : PR_IntervalToMilliseconds(clockEnd - clockStart)));
827 : }
828 :
829 : return rv;
830 : }
831 :
832 : nsresult
833 0 : Classifier::ApplyUpdatesForeground(nsresult aBackgroundRv,
834 : const nsACString& aFailedTableName)
835 : {
836 0 : if (mUpdateInterrupted) {
837 0 : LOG(("Update is interrupted! Just remove update intermediaries."));
838 0 : RemoveUpdateIntermediaries();
839 0 : return NS_OK;
840 : }
841 0 : if (NS_SUCCEEDED(aBackgroundRv)) {
842 : // Copy and Invalidate fullhash cache here because this call requires
843 : // mLookupCaches which is only available on work-thread
844 0 : CopyAndInvalidateFullHashCache();
845 :
846 0 : return SwapInNewTablesAndCleanup();
847 : }
848 0 : if (NS_ERROR_OUT_OF_MEMORY != aBackgroundRv) {
849 0 : ResetTables(Clear_All, nsTArray<nsCString> { nsCString(aFailedTableName) });
850 : }
851 : return aBackgroundRv;
852 : }
853 :
854 : nsresult
855 0 : Classifier::ApplyFullHashes(ConstTableUpdateArray& aUpdates)
856 : {
857 0 : MOZ_ASSERT(NS_GetCurrentThread() != mUpdateThread,
858 : "ApplyFullHashes() MUST NOT be called on update thread");
859 0 : MOZ_ASSERT(!NS_IsMainThread(),
860 : "ApplyFullHashes() must be called on the classifier worker thread.");
861 :
862 0 : LOG(("Applying %zu table gethashes.", aUpdates.Length()));
863 :
864 0 : for (uint32_t i = 0; i < aUpdates.Length(); i++) {
865 0 : nsresult rv = UpdateCache(aUpdates[i]);
866 0 : NS_ENSURE_SUCCESS(rv, rv);
867 :
868 0 : aUpdates[i] = nullptr;
869 : }
870 :
871 : return NS_OK;
872 : }
873 :
874 : void
875 0 : Classifier::GetCacheInfo(const nsACString& aTable,
876 : nsIUrlClassifierCacheInfo** aCache)
877 : {
878 0 : RefPtr<const LookupCache> lookupCache = GetLookupCache(aTable);
879 0 : if (!lookupCache) {
880 0 : return;
881 : }
882 :
883 0 : lookupCache->GetCacheInfo(aCache);
884 : }
885 :
886 : void
887 0 : Classifier::DropStores()
888 : {
889 : // See the comment in Classifier::Close() before adding anything here.
890 0 : mLookupCaches.Clear();
891 0 : }
892 :
893 : nsresult
894 1 : Classifier::RegenActiveTables()
895 : {
896 1 : mActiveTablesCache.Clear();
897 :
898 0 : nsTArray<nsCString> foundTables;
899 0 : ScanStoreDir(mRootStoreDirectory, foundTables);
900 :
901 2 : for (uint32_t i = 0; i < foundTables.Length(); i++) {
902 0 : nsCString table(foundTables[i]);
903 :
904 0 : RefPtr<const LookupCache> lookupCache = GetLookupCache(table);
905 0 : if (!lookupCache) {
906 0 : LOG(("Inactive table (no cache): %s", table.get()));
907 0 : continue;
908 : }
909 :
910 0 : if (!lookupCache->IsPrimed()) {
911 0 : LOG(("Inactive table (cache not primed): %s", table.get()));
912 : continue;
913 : }
914 :
915 0 : if (LookupCache::Cast<const LookupCacheV4>(lookupCache)) {
916 0 : LOG(("Active v4 table: %s", table.get()));
917 : } else {
918 0 : HashStore store(table, GetProvider(table), mRootStoreDirectory);
919 :
920 0 : nsresult rv = store.Open();
921 0 : if (NS_FAILED(rv)) {
922 0 : continue;
923 : }
924 :
925 0 : const ChunkSet &adds = store.AddChunks();
926 0 : const ChunkSet &subs = store.SubChunks();
927 :
928 0 : if (adds.Length() == 0 && subs.Length() == 0) {
929 : continue;
930 : }
931 :
932 0 : LOG(("Active v2 table: %s", store.TableName().get()));
933 : }
934 :
935 0 : mActiveTablesCache.AppendElement(table);
936 : }
937 :
938 0 : return NS_OK;
939 : }
940 :
941 : nsresult
942 0 : Classifier::ScanStoreDir(nsIFile* aDirectory, nsTArray<nsCString>& aTables)
943 : {
944 0 : nsCOMPtr<nsIDirectoryEnumerator> entries;
945 1 : nsresult rv = aDirectory->GetDirectoryEntries(getter_AddRefs(entries));
946 1 : NS_ENSURE_SUCCESS(rv, rv);
947 :
948 0 : nsCOMPtr<nsIFile> file;
949 3 : while (NS_SUCCEEDED(rv = entries->GetNextFile(getter_AddRefs(file))) && file) {
950 : // If |file| is a directory, recurse to find its entries as well.
951 : bool isDirectory;
952 0 : if (NS_FAILED(file->IsDirectory(&isDirectory))) {
953 0 : continue;
954 : }
955 0 : if (isDirectory) {
956 0 : ScanStoreDir(file, aTables);
957 0 : continue;
958 : }
959 :
960 0 : nsCString leafName;
961 0 : rv = file->GetNativeLeafName(leafName);
962 0 : NS_ENSURE_SUCCESS(rv, rv);
963 :
964 : // Both v2 and v4 contain .pset file
965 0 : nsCString suffix(NS_LITERAL_CSTRING(".pset"));
966 :
967 0 : int32_t dot = leafName.RFind(suffix);
968 0 : if (dot != -1) {
969 0 : leafName.Cut(dot, suffix.Length());
970 0 : aTables.AppendElement(leafName);
971 : }
972 : }
973 1 : NS_ENSURE_SUCCESS(rv, rv);
974 :
975 : return NS_OK;
976 : }
977 :
978 : nsresult
979 0 : Classifier::ActiveTables(nsTArray<nsCString>& aTables) const
980 : {
981 0 : aTables = mActiveTablesCache;
982 0 : return NS_OK;
983 : }
984 :
985 : nsresult
986 1 : Classifier::CleanToDelete()
987 : {
988 : bool exists;
989 0 : nsresult rv = mToDeleteDirectory->Exists(&exists);
990 0 : NS_ENSURE_SUCCESS(rv, rv);
991 :
992 0 : if (exists) {
993 0 : rv = mToDeleteDirectory->Remove(true);
994 0 : NS_ENSURE_SUCCESS(rv, rv);
995 : }
996 :
997 : return NS_OK;
998 : }
999 :
1000 : #ifdef MOZ_SAFEBROWSING_DUMP_FAILED_UPDATES
1001 :
1002 : already_AddRefed<nsIFile>
1003 0 : Classifier::GetFailedUpdateDirectroy()
1004 : {
1005 0 : nsCString failedUpdatekDirName = STORE_DIRECTORY + nsCString("-failedupdate");
1006 :
1007 0 : nsCOMPtr<nsIFile> failedUpdatekDirectory;
1008 0 : if (NS_FAILED(mCacheDirectory->Clone(getter_AddRefs(failedUpdatekDirectory))) ||
1009 0 : NS_FAILED(failedUpdatekDirectory->AppendNative(failedUpdatekDirName))) {
1010 0 : LOG(("Failed to init failedUpdatekDirectory."));
1011 : return nullptr;
1012 : }
1013 :
1014 0 : return failedUpdatekDirectory.forget();
1015 : }
1016 :
1017 : nsresult
1018 0 : Classifier::DumpRawTableUpdates(const nsACString& aRawUpdates)
1019 : {
1020 0 : LOG(("Dumping raw table updates..."));
1021 :
1022 0 : DumpFailedUpdate();
1023 :
1024 0 : nsCOMPtr<nsIFile> failedUpdatekDirectory = GetFailedUpdateDirectroy();
1025 :
1026 : // Create tableupdate.bin and dump raw table update data.
1027 0 : nsCOMPtr<nsIFile> rawTableUpdatesFile;
1028 0 : nsCOMPtr<nsIOutputStream> outputStream;
1029 0 : if (NS_FAILED(failedUpdatekDirectory->Clone(getter_AddRefs(rawTableUpdatesFile))) ||
1030 0 : NS_FAILED(rawTableUpdatesFile->AppendNative(nsCString("tableupdates.bin"))) ||
1031 0 : NS_FAILED(NS_NewLocalFileOutputStream(getter_AddRefs(outputStream),
1032 : rawTableUpdatesFile,
1033 : PR_WRONLY | PR_TRUNCATE | PR_CREATE_FILE))) {
1034 0 : LOG(("Failed to create file to dump raw table updates."));
1035 : return NS_ERROR_FAILURE;
1036 : }
1037 :
1038 : // Write out the data.
1039 : uint32_t written;
1040 0 : nsresult rv = outputStream->Write(aRawUpdates.BeginReading(),
1041 0 : aRawUpdates.Length(), &written);
1042 0 : NS_ENSURE_SUCCESS(rv, rv);
1043 0 : NS_ENSURE_TRUE(written == aRawUpdates.Length(), NS_ERROR_FAILURE);
1044 :
1045 : return rv;
1046 : }
1047 :
1048 : nsresult
1049 0 : Classifier::DumpFailedUpdate()
1050 : {
1051 0 : LOG(("Dumping failed update..."));
1052 :
1053 0 : nsCOMPtr<nsIFile> failedUpdatekDirectory = GetFailedUpdateDirectroy();
1054 :
1055 : // Remove the "failed update" directory no matter it exists or not.
1056 : // Failure is fine because the directory may not exist.
1057 0 : failedUpdatekDirectory->Remove(true);
1058 :
1059 0 : nsCString failedUpdatekDirName;
1060 0 : nsresult rv = failedUpdatekDirectory->GetNativeLeafName(failedUpdatekDirName);
1061 0 : NS_ENSURE_SUCCESS(rv, rv);
1062 :
1063 : // Copy the in-use directory to a clean "failed update" directory.
1064 0 : nsCOMPtr<nsIFile> inUseDirectory;
1065 0 : if (NS_FAILED(mRootStoreDirectory->Clone(getter_AddRefs(inUseDirectory))) ||
1066 0 : NS_FAILED(inUseDirectory->CopyToNative(nullptr, failedUpdatekDirName))) {
1067 0 : LOG(("Failed to move in-use to the \"failed update\" directory %s",
1068 : failedUpdatekDirName.get()));
1069 : return NS_ERROR_FAILURE;
1070 : }
1071 :
1072 : return rv;
1073 : }
1074 :
1075 : #endif // MOZ_SAFEBROWSING_DUMP_FAILED_UPDATES
1076 :
1077 : nsresult
1078 0 : Classifier::CopyInUseDirForUpdate()
1079 : {
1080 0 : LOG(("Copy in-use directory content for update."));
1081 :
1082 : // We copy everything from in-use directory to a temporary directory
1083 : // for updating.
1084 :
1085 0 : nsCString updatingDirName;
1086 0 : nsresult rv = mUpdatingDirectory->GetNativeLeafName(updatingDirName);
1087 0 : NS_ENSURE_SUCCESS(rv, rv);
1088 :
1089 : // Remove the destination directory first (just in case) the do the copy.
1090 0 : mUpdatingDirectory->Remove(true);
1091 0 : if (!mRootStoreDirectoryForUpdate) {
1092 0 : LOG(("mRootStoreDirectoryForUpdate is null."));
1093 : return NS_ERROR_NULL_POINTER;
1094 : }
1095 0 : rv = mRootStoreDirectoryForUpdate->CopyToNative(nullptr, updatingDirName);
1096 0 : NS_ENSURE_SUCCESS(rv, rv);
1097 :
1098 : return NS_OK;
1099 : }
1100 :
1101 : nsresult
1102 0 : Classifier::RecoverBackups()
1103 : {
1104 : bool backupExists;
1105 1 : nsresult rv = mBackupDirectory->Exists(&backupExists);
1106 1 : NS_ENSURE_SUCCESS(rv, rv);
1107 :
1108 0 : if (backupExists) {
1109 : // Remove the safebrowsing dir if it exists
1110 0 : nsCString storeDirName;
1111 0 : rv = mRootStoreDirectory->GetNativeLeafName(storeDirName);
1112 0 : NS_ENSURE_SUCCESS(rv, rv);
1113 :
1114 : bool storeExists;
1115 0 : rv = mRootStoreDirectory->Exists(&storeExists);
1116 0 : NS_ENSURE_SUCCESS(rv, rv);
1117 :
1118 0 : if (storeExists) {
1119 0 : rv = mRootStoreDirectory->Remove(true);
1120 0 : NS_ENSURE_SUCCESS(rv, rv);
1121 : }
1122 :
1123 : // Move the backup to the store location
1124 0 : rv = mBackupDirectory->MoveToNative(nullptr, storeDirName);
1125 0 : NS_ENSURE_SUCCESS(rv, rv);
1126 :
1127 : // mBackupDirectory now points to storeDir, fix up.
1128 0 : rv = SetupPathNames();
1129 0 : NS_ENSURE_SUCCESS(rv, rv);
1130 : }
1131 :
1132 : return NS_OK;
1133 : }
1134 :
1135 : bool
1136 0 : Classifier::CheckValidUpdate(TableUpdateArray& aUpdates,
1137 : const nsACString& aTable)
1138 : {
1139 : // take the quick exit if there is no valid update for us
1140 : // (common case)
1141 0 : uint32_t validupdates = 0;
1142 :
1143 0 : for (uint32_t i = 0; i < aUpdates.Length(); i++) {
1144 0 : RefPtr<const TableUpdate> update = aUpdates[i];
1145 0 : if (!update || !update->TableName().Equals(aTable)) {
1146 0 : continue;
1147 : }
1148 0 : if (update->Empty()) {
1149 0 : aUpdates[i] = nullptr;
1150 0 : continue;
1151 : }
1152 0 : validupdates++;
1153 : }
1154 :
1155 0 : if (!validupdates) {
1156 : // This can happen if the update was only valid for one table.
1157 : return false;
1158 : }
1159 :
1160 0 : return true;
1161 : }
1162 :
1163 : nsCString
1164 0 : Classifier::GetProvider(const nsACString& aTableName)
1165 : {
1166 : nsCOMPtr<nsIUrlClassifierUtils> urlUtil =
1167 0 : do_GetService(NS_URLCLASSIFIERUTILS_CONTRACTID);
1168 :
1169 0 : nsCString provider;
1170 0 : nsresult rv = urlUtil->GetProvider(aTableName, provider);
1171 :
1172 0 : return NS_SUCCEEDED(rv) ? provider : EmptyCString();
1173 : }
1174 :
1175 : /*
1176 : * This will consume+delete updates from the passed nsTArray.
1177 : */
1178 : nsresult
1179 0 : Classifier::UpdateHashStore(TableUpdateArray& aUpdates,
1180 : const nsACString& aTable)
1181 : {
1182 0 : if (nsUrlClassifierDBService::ShutdownHasStarted()) {
1183 : return NS_ERROR_UC_UPDATE_SHUTDOWNING;
1184 : }
1185 :
1186 0 : LOG(("Classifier::UpdateHashStore(%s)", PromiseFlatCString(aTable).get()));
1187 :
1188 0 : HashStore store(aTable, GetProvider(aTable), mUpdatingDirectory);
1189 :
1190 0 : if (!CheckValidUpdate(aUpdates, store.TableName())) {
1191 : return NS_OK;
1192 : }
1193 :
1194 0 : nsresult rv = store.Open();
1195 0 : NS_ENSURE_SUCCESS(rv, rv);
1196 0 : rv = store.BeginUpdate();
1197 0 : NS_ENSURE_SUCCESS(rv, rv);
1198 :
1199 : // Read the part of the store that is (only) in the cache
1200 0 : RefPtr<LookupCacheV2> lookupCacheV2;
1201 : {
1202 0 : RefPtr<LookupCache> lookupCache = GetLookupCacheForUpdate(store.TableName());
1203 0 : if (lookupCache) {
1204 0 : lookupCacheV2 = LookupCache::Cast<LookupCacheV2>(lookupCache);
1205 : }
1206 : }
1207 0 : if (!lookupCacheV2) {
1208 : return NS_ERROR_UC_UPDATE_TABLE_NOT_FOUND;
1209 : }
1210 :
1211 0 : FallibleTArray<uint32_t> AddPrefixHashes;
1212 0 : rv = lookupCacheV2->GetPrefixes(AddPrefixHashes);
1213 0 : NS_ENSURE_SUCCESS(rv, rv);
1214 0 : rv = store.AugmentAdds(AddPrefixHashes);
1215 0 : NS_ENSURE_SUCCESS(rv, rv);
1216 0 : AddPrefixHashes.Clear();
1217 :
1218 0 : uint32_t applied = 0;
1219 :
1220 0 : for (uint32_t i = 0; i < aUpdates.Length(); i++) {
1221 0 : RefPtr<TableUpdate> update = aUpdates[i];
1222 0 : if (!update || !update->TableName().Equals(store.TableName())) {
1223 0 : continue;
1224 : }
1225 :
1226 0 : RefPtr<TableUpdateV2> updateV2 = TableUpdate::Cast<TableUpdateV2>(update);
1227 0 : NS_ENSURE_TRUE(updateV2, NS_ERROR_UC_UPDATE_UNEXPECTED_VERSION);
1228 :
1229 0 : rv = store.ApplyUpdate(updateV2);
1230 0 : NS_ENSURE_SUCCESS(rv, rv);
1231 :
1232 0 : applied++;
1233 :
1234 0 : LOG(("Applied update to table %s:", store.TableName().get()));
1235 0 : LOG((" %d add chunks", updateV2->AddChunks().Length()));
1236 0 : LOG((" %zu add prefixes", updateV2->AddPrefixes().Length()));
1237 0 : LOG((" %zu add completions", updateV2->AddCompletes().Length()));
1238 0 : LOG((" %d sub chunks", updateV2->SubChunks().Length()));
1239 0 : LOG((" %zu sub prefixes", updateV2->SubPrefixes().Length()));
1240 0 : LOG((" %zu sub completions", updateV2->SubCompletes().Length()));
1241 0 : LOG((" %d add expirations", updateV2->AddExpirations().Length()));
1242 0 : LOG((" %d sub expirations", updateV2->SubExpirations().Length()));
1243 :
1244 0 : aUpdates[i] = nullptr;
1245 : }
1246 :
1247 0 : LOG(("Applied %d update(s) to %s.", applied, store.TableName().get()));
1248 :
1249 0 : rv = store.Rebuild();
1250 0 : NS_ENSURE_SUCCESS(rv, rv);
1251 :
1252 0 : LOG(("Table %s now has:", store.TableName().get()));
1253 0 : LOG((" %d add chunks", store.AddChunks().Length()));
1254 0 : LOG((" %zu add prefixes", store.AddPrefixes().Length()));
1255 0 : LOG((" %zu add completions", store.AddCompletes().Length()));
1256 0 : LOG((" %d sub chunks", store.SubChunks().Length()));
1257 0 : LOG((" %zu sub prefixes", store.SubPrefixes().Length()));
1258 0 : LOG((" %zu sub completions", store.SubCompletes().Length()));
1259 :
1260 0 : rv = store.WriteFile();
1261 0 : NS_ENSURE_SUCCESS(rv, rv);
1262 :
1263 : // At this point the store is updated and written out to disk, but
1264 : // the data is still in memory. Build our quick-lookup table here.
1265 0 : rv = lookupCacheV2->Build(store.AddPrefixes(), store.AddCompletes());
1266 0 : NS_ENSURE_SUCCESS(rv, NS_ERROR_UC_UPDATE_BUILD_PREFIX_FAILURE);
1267 :
1268 : #if defined(DEBUG)
1269 0 : lookupCacheV2->DumpCompletions();
1270 : #endif
1271 0 : rv = lookupCacheV2->WriteFile();
1272 0 : NS_ENSURE_SUCCESS(rv, NS_ERROR_UC_UPDATE_FAIL_TO_WRITE_DISK);
1273 :
1274 0 : LOG(("Successfully updated %s", store.TableName().get()));
1275 :
1276 : return NS_OK;
1277 : }
1278 :
1279 : nsresult
1280 0 : Classifier::UpdateTableV4(TableUpdateArray& aUpdates,
1281 : const nsACString& aTable)
1282 : {
1283 0 : MOZ_ASSERT(!NS_IsMainThread(),
1284 : "UpdateTableV4 must be called on the classifier worker thread.");
1285 0 : if (nsUrlClassifierDBService::ShutdownHasStarted()) {
1286 : return NS_ERROR_UC_UPDATE_SHUTDOWNING;
1287 : }
1288 :
1289 0 : LOG(("Classifier::UpdateTableV4(%s)", PromiseFlatCString(aTable).get()));
1290 :
1291 0 : if (!CheckValidUpdate(aUpdates, aTable)) {
1292 : return NS_OK;
1293 : }
1294 :
1295 0 : RefPtr<LookupCacheV4> lookupCacheV4;
1296 : {
1297 0 : RefPtr<LookupCache> lookupCache = GetLookupCacheForUpdate(aTable);
1298 0 : if (lookupCache) {
1299 0 : lookupCacheV4 = LookupCache::Cast<LookupCacheV4>(lookupCache);
1300 : }
1301 : }
1302 0 : if (!lookupCacheV4) {
1303 : return NS_ERROR_UC_UPDATE_TABLE_NOT_FOUND;
1304 : }
1305 :
1306 0 : nsresult rv = NS_OK;
1307 :
1308 : // If there are multiple updates for the same table, prefixes1 & prefixes2
1309 : // will act as input and output in turn to reduce memory copy overhead.
1310 0 : PrefixStringMap prefixes1, prefixes2;
1311 0 : PrefixStringMap* input = &prefixes1;
1312 0 : PrefixStringMap* output = &prefixes2;
1313 :
1314 0 : RefPtr<const TableUpdateV4> lastAppliedUpdate = nullptr;
1315 0 : for (uint32_t i = 0; i < aUpdates.Length(); i++) {
1316 0 : RefPtr<TableUpdate> update = aUpdates[i];
1317 0 : if (!update || !update->TableName().Equals(aTable)) {
1318 0 : continue;
1319 : }
1320 :
1321 0 : RefPtr<TableUpdateV4> updateV4 = TableUpdate::Cast<TableUpdateV4>(update);
1322 0 : NS_ENSURE_TRUE(updateV4, NS_ERROR_UC_UPDATE_UNEXPECTED_VERSION);
1323 :
1324 0 : if (updateV4->IsFullUpdate()) {
1325 0 : input->Clear();
1326 0 : output->Clear();
1327 0 : rv = lookupCacheV4->ApplyUpdate(updateV4, *input, *output);
1328 0 : if (NS_FAILED(rv)) {
1329 : return rv;
1330 : }
1331 : } else {
1332 : // If both prefix sets are empty, this means we are doing a partial update
1333 : // without a prior full/partial update in the loop. In this case we should
1334 : // get prefixes from the lookup cache first.
1335 0 : if (prefixes1.IsEmpty() && prefixes2.IsEmpty()) {
1336 0 : lookupCacheV4->GetPrefixes(prefixes1);
1337 : } else {
1338 0 : MOZ_ASSERT(prefixes1.IsEmpty() ^ prefixes2.IsEmpty());
1339 :
1340 : // When there are multiple partial updates, input should always point
1341 : // to the non-empty prefix set(filled by previous full/partial update).
1342 : // output should always point to the empty prefix set.
1343 0 : input = prefixes1.IsEmpty() ? &prefixes2 : &prefixes1;
1344 0 : output = prefixes1.IsEmpty() ? &prefixes1 : &prefixes2;
1345 : }
1346 :
1347 0 : rv = lookupCacheV4->ApplyUpdate(updateV4, *input, *output);
1348 0 : if (NS_FAILED(rv)) {
1349 : return rv;
1350 : }
1351 :
1352 0 : input->Clear();
1353 : }
1354 :
1355 : // Keep track of the last applied update.
1356 0 : lastAppliedUpdate = updateV4;
1357 :
1358 0 : aUpdates[i] = nullptr;
1359 : }
1360 :
1361 0 : rv = lookupCacheV4->Build(*output);
1362 0 : NS_ENSURE_SUCCESS(rv, NS_ERROR_UC_UPDATE_BUILD_PREFIX_FAILURE);
1363 :
1364 0 : rv = lookupCacheV4->WriteFile();
1365 0 : NS_ENSURE_SUCCESS(rv, NS_ERROR_UC_UPDATE_FAIL_TO_WRITE_DISK);
1366 :
1367 0 : if (lastAppliedUpdate) {
1368 0 : LOG(("Write meta data of the last applied update."));
1369 0 : rv = lookupCacheV4->WriteMetadata(lastAppliedUpdate);
1370 0 : NS_ENSURE_SUCCESS(rv, NS_ERROR_UC_UPDATE_FAIL_TO_WRITE_DISK);
1371 : }
1372 :
1373 0 : LOG(("Successfully updated %s\n", PromiseFlatCString(aTable).get()));
1374 :
1375 : return NS_OK;
1376 : }
1377 :
1378 : nsresult
1379 0 : Classifier::UpdateCache(RefPtr<const TableUpdate> aUpdate)
1380 : {
1381 0 : if (!aUpdate) {
1382 : return NS_OK;
1383 : }
1384 :
1385 0 : nsAutoCString table(aUpdate->TableName());
1386 0 : LOG(("Classifier::UpdateCache(%s)", table.get()));
1387 :
1388 0 : RefPtr<LookupCache> lookupCache = GetLookupCache(table);
1389 0 : if (!lookupCache) {
1390 : return NS_ERROR_FAILURE;
1391 : }
1392 :
1393 0 : RefPtr<LookupCacheV2> lookupV2 = LookupCache::Cast<LookupCacheV2>(lookupCache);
1394 0 : if (lookupV2) {
1395 0 : RefPtr<const TableUpdateV2> updateV2 = TableUpdate::Cast<TableUpdateV2>(aUpdate);
1396 0 : lookupV2->AddGethashResultToCache(updateV2->AddCompletes(),
1397 0 : updateV2->MissPrefixes());
1398 : } else {
1399 0 : RefPtr<LookupCacheV4> lookupV4 = LookupCache::Cast<LookupCacheV4>(lookupCache);
1400 0 : if (!lookupV4) {
1401 0 : return NS_ERROR_FAILURE;
1402 : }
1403 :
1404 0 : RefPtr<const TableUpdateV4> updateV4 = TableUpdate::Cast<TableUpdateV4>(aUpdate);
1405 0 : lookupV4->AddFullHashResponseToCache(updateV4->FullHashResponse());
1406 : }
1407 :
1408 : #if defined(DEBUG)
1409 0 : lookupCache->DumpCache();
1410 : #endif
1411 :
1412 0 : return NS_OK;
1413 : }
1414 :
1415 : RefPtr<LookupCache>
1416 0 : Classifier::GetLookupCache(const nsACString& aTable, bool aForUpdate)
1417 : {
1418 : // GetLookupCache(aForUpdate==true) can only be called on update thread.
1419 0 : MOZ_ASSERT_IF(aForUpdate, NS_GetCurrentThread() == mUpdateThread);
1420 :
1421 : LookupCacheArray& lookupCaches = aForUpdate ? mNewLookupCaches
1422 0 : : mLookupCaches;
1423 : auto& rootStoreDirectory = aForUpdate ? mUpdatingDirectory
1424 0 : : mRootStoreDirectory;
1425 :
1426 0 : for (auto c: lookupCaches) {
1427 0 : if (c->TableName().Equals(aTable)) {
1428 0 : return c;
1429 : }
1430 : }
1431 :
1432 : // We don't want to create lookupcache when shutdown is already happening.
1433 0 : if (nsUrlClassifierDBService::ShutdownHasStarted()) {
1434 : return nullptr;
1435 : }
1436 :
1437 : // TODO : Bug 1302600, It would be better if we have a more general non-main
1438 : // thread method to convert table name to protocol version. Currently
1439 : // we can only know this by checking if the table name ends with '-proto'.
1440 0 : RefPtr<LookupCache> cache;
1441 0 : nsCString provider = GetProvider(aTable);
1442 0 : if (StringEndsWith(aTable, NS_LITERAL_CSTRING("-proto"))) {
1443 0 : cache = new LookupCacheV4(aTable, provider, rootStoreDirectory);
1444 : } else {
1445 0 : cache = new LookupCacheV2(aTable, provider, rootStoreDirectory);
1446 : }
1447 :
1448 0 : nsresult rv = cache->Init();
1449 0 : if (NS_FAILED(rv)) {
1450 : return nullptr;
1451 : }
1452 0 : rv = cache->Open();
1453 0 : if (NS_SUCCEEDED(rv)) {
1454 0 : lookupCaches.AppendElement(cache);
1455 : return cache;
1456 : }
1457 :
1458 : // At this point we failed to open LookupCache.
1459 : //
1460 : // GetLookupCache for update and for other usage will run on update thread
1461 : // and worker thread respectively (Bug 1339760). Removing stuff only in
1462 : // their own realms potentially increases the concurrency.
1463 :
1464 0 : if (aForUpdate) {
1465 : // Remove intermediaries no matter if it's due to file corruption or not.
1466 0 : RemoveUpdateIntermediaries();
1467 : return nullptr;
1468 : }
1469 :
1470 : // Non-update case.
1471 0 : if (rv == NS_ERROR_FILE_CORRUPTED) {
1472 0 : Reset(); // Not including the update intermediaries.
1473 : }
1474 : return nullptr;
1475 : }
1476 :
1477 : nsresult
1478 0 : Classifier::ReadNoiseEntries(const Prefix& aPrefix,
1479 : const nsACString& aTableName,
1480 : uint32_t aCount,
1481 : PrefixArray& aNoiseEntries)
1482 : {
1483 0 : FallibleTArray<uint32_t> prefixes;
1484 : nsresult rv;
1485 :
1486 0 : RefPtr<LookupCache> cache = GetLookupCache(aTableName);
1487 0 : if (!cache) {
1488 : return NS_ERROR_FAILURE;
1489 : }
1490 :
1491 0 : RefPtr<LookupCacheV2> cacheV2 = LookupCache::Cast<LookupCacheV2>(cache);
1492 0 : if (cacheV2) {
1493 0 : rv = cacheV2->GetPrefixes(prefixes);
1494 : } else {
1495 0 : rv = LookupCache::Cast<LookupCacheV4>(cache)->GetFixedLengthPrefixes(prefixes);
1496 : }
1497 :
1498 0 : NS_ENSURE_SUCCESS(rv, rv);
1499 :
1500 0 : if (prefixes.Length() == 0) {
1501 0 : NS_WARNING("Could not find prefix in PrefixSet during noise lookup");
1502 0 : return NS_ERROR_FAILURE;
1503 : }
1504 :
1505 : // We do not want to simply pick random prefixes, because this would allow
1506 : // averaging out the noise by analysing the traffic from Firefox users.
1507 : // Instead, we ensure the 'noise' is the same for the same prefix by seeding
1508 : // the random number generator with the prefix. We prefer not to use rand()
1509 : // which isn't thread safe, and the reseeding of which could trip up other
1510 : // parts othe code that expect actual random numbers.
1511 : // Here we use a simple LCG (Linear Congruential Generator) to generate
1512 : // random numbers. We seed the LCG with the prefix we are generating noise
1513 : // for.
1514 : // http://en.wikipedia.org/wiki/Linear_congruential_generator
1515 :
1516 0 : uint32_t m = prefixes.Length();
1517 0 : uint32_t a = aCount % m;
1518 0 : uint32_t idx = aPrefix.ToUint32() % m;
1519 :
1520 0 : for (size_t i = 0; i < aCount; i++) {
1521 0 : idx = (a * idx + a) % m;
1522 :
1523 : Prefix newPrefix;
1524 0 : uint32_t hash = prefixes[idx];
1525 : // In the case V4 little endian, we did swapping endian when converting from char* to
1526 : // int, should revert endian to make sure we will send hex string correctly
1527 : // See https://bugzilla.mozilla.org/show_bug.cgi?id=1283007#c23
1528 0 : if (!cacheV2 && !bool(MOZ_BIG_ENDIAN)) {
1529 0 : hash = NativeEndian::swapFromBigEndian(prefixes[idx]);
1530 : }
1531 :
1532 0 : newPrefix.FromUint32(hash);
1533 0 : if (newPrefix != aPrefix) {
1534 0 : aNoiseEntries.AppendElement(newPrefix);
1535 : }
1536 : }
1537 :
1538 : return NS_OK;
1539 : }
1540 :
1541 : nsresult
1542 0 : Classifier::LoadMetadata(nsIFile* aDirectory, nsACString& aResult)
1543 : {
1544 0 : nsCOMPtr<nsIDirectoryEnumerator> entries;
1545 0 : nsresult rv = aDirectory->GetDirectoryEntries(getter_AddRefs(entries));
1546 0 : NS_ENSURE_SUCCESS(rv, rv);
1547 0 : NS_ENSURE_ARG_POINTER(entries);
1548 :
1549 0 : nsCOMPtr<nsIFile> file;
1550 0 : while (NS_SUCCEEDED(rv = entries->GetNextFile(getter_AddRefs(file))) && file) {
1551 : // If |file| is a directory, recurse to find its entries as well.
1552 : bool isDirectory;
1553 0 : if (NS_FAILED(file->IsDirectory(&isDirectory))) {
1554 0 : continue;
1555 : }
1556 0 : if (isDirectory) {
1557 0 : LoadMetadata(file, aResult);
1558 0 : continue;
1559 : }
1560 :
1561 : // Truncate file extension to get the table name.
1562 0 : nsCString tableName;
1563 0 : rv = file->GetNativeLeafName(tableName);
1564 0 : NS_ENSURE_SUCCESS(rv, rv);
1565 :
1566 0 : int32_t dot = tableName.RFind(METADATA_SUFFIX);
1567 0 : if (dot == -1) {
1568 : continue;
1569 : }
1570 0 : tableName.Cut(dot, METADATA_SUFFIX.Length());
1571 :
1572 0 : RefPtr<LookupCacheV4> lookupCacheV4;
1573 : {
1574 0 : RefPtr<LookupCache> lookupCache = GetLookupCache(tableName);
1575 0 : if (lookupCache) {
1576 0 : lookupCacheV4 = LookupCache::Cast<LookupCacheV4>(lookupCache);
1577 : }
1578 : }
1579 0 : if (!lookupCacheV4) {
1580 0 : continue;
1581 : }
1582 :
1583 0 : nsCString state;
1584 0 : nsCString checksum;
1585 0 : rv = lookupCacheV4->LoadMetadata(state, checksum);
1586 0 : if (NS_FAILED(rv)) {
1587 0 : LOG(("Failed to get metadata for table %s", tableName.get()));
1588 0 : continue;
1589 : }
1590 :
1591 : // The state might include '\n' so that we have to encode.
1592 0 : nsAutoCString stateBase64;
1593 0 : rv = Base64Encode(state, stateBase64);
1594 0 : NS_ENSURE_SUCCESS(rv, rv);
1595 :
1596 0 : nsAutoCString checksumBase64;
1597 0 : rv = Base64Encode(checksum, checksumBase64);
1598 0 : NS_ENSURE_SUCCESS(rv, rv);
1599 :
1600 0 : LOG(("Appending state '%s' and checksum '%s' for table %s",
1601 : stateBase64.get(), checksumBase64.get(), tableName.get()));
1602 :
1603 0 : aResult.AppendPrintf("%s;%s:%s\n", tableName.get(),
1604 : stateBase64.get(),
1605 0 : checksumBase64.get());
1606 : }
1607 :
1608 : return rv;
1609 : }
1610 :
1611 : } // namespace safebrowsing
1612 : } // namespace mozilla
|
