Performance improvements (#3)

* Amazon multithreaded search * Changing products search names * # Conflicts: # src/main/kotlin/Launcher.kt # src/main/kotlin/MainActivityPresenter.kt # src/main/kotlin/usecases/AmazonSearchProduct.kt * Fixing multithreading for be compatible with fnac use case * Making CorteIngles to be multithreaded * Improving buzzwords * Closing chrome when it ends corte ingles
ConradoMateu · Nov 12, 2017 · 5a8d824 · 5a8d824
1 parent 2722ffb
commit 5a8d824
Show file tree

Hide file tree

Showing 9 changed files with 66 additions and 37 deletions.
diff --git a/build.gradle b/build.gradle
@@ -37,9 +37,17 @@ dependencies {
     compile "org.jetbrains.kotlin:kotlin-stdlib-jre8:$kotlin_version"
 
     compile 'no.tornado:tornadofx:1.7.11'
+    compile 'org.jetbrains.kotlinx:kotlinx-coroutines-core:0.19.3'
+    compile 'org.jetbrains.kotlinx:kotlinx-coroutines-javafx:0.19.3'
 
     compile 'com.github.salomonbrys.kodein:kodein:4.1.0'
     compile 'org.seleniumhq.selenium:selenium-java:3.0.1'
 
     testCompile 'junit:junit:4.12'
+}
+
+kotlin {
+    experimental {
+        coroutines "enable"
+    }
 }
diff --git a/src/main/kotlin/Constants.kt b/src/main/kotlin/Constants.kt
@@ -26,6 +26,6 @@ object CONSTANTS{
     }
 
     object CORTEINGLES{
-        val URL = "https://elcorteingles.es"
+        val URL = "https://www.elcorteingles.es"
     }
 }
diff --git a/src/main/kotlin/MainActivityPresenter.kt b/src/main/kotlin/MainActivityPresenter.kt
@@ -1,5 +1,8 @@
 import di.kdi
 import domain.Product
+import kotlinx.coroutines.experimental.async
+import kotlinx.coroutines.experimental.javafx.JavaFx
+import kotlinx.coroutines.experimental.launch
 import tornadofx.Controller
 import tornadofx.observable
 import usecases.*
@@ -25,35 +28,36 @@ class MainActivityPresenter : Controller() {
         val selectedStores = stores.filter { (_, value) -> value }.map { (key, _) -> key }
         val products = mutableListOf<Product>()
 
-        if (selectedStores.contains("https://www.amazon.es")) {
-            searchItemsForStore(amazonSearchProduct, brands, article, pages, products)
-        }
-
-        if (selectedStores.contains("https://www.fnac.es")) {
-            searchItemsForStore(fnacSearchProduct, brands, article, pages, products)
-        }
+        launch(JavaFx) {
+            if (brands.isNotEmpty()) {
+                selectedStores.forEach { store ->
+                    brands.map { brand ->
+                        async { searchItemsForStore(store, brand, article, pages, products) }
+                    }.map { it.await() }
+                }
+            } else {
+                selectedStores.forEach {
+                    async { searchItemsForStore(it, null, article, pages, products) }.await()
+                }
+            }
+
+            ResultsActivity.navigateWithPreviousResults()
 
-        if (selectedStores.contains("https://www.elcorteingles.es")) {
-            searchItemsForStore(corteInglesSearchProduct, brands, article, pages, products)
         }
 
-        ResultsActivity.navigateWithPreviousResults()
-
     }
 
-    private fun searchItemsForStore(storeUseCase: ISearchProducts, brands: List<String>, article: String, pages: Int, products: MutableList<Product>) {
-        if (brands.isNotEmpty()) {
-            val storeProducts = brands
-                    .map { brand: String ->
-                        storeUseCase(if (article == articles[0]) "Cafetera" else article, brand, pages)
-                    }.flatten()
-            addProcessedProducts(storeProducts)
-            products.addAll(storeProducts)
-        } else {
-            val storeProducts = storeUseCase(if (article == articles[0]) "Cafetera" else article, page = pages)
-            addProcessedProducts(storeProducts)
-            products.addAll(storeProducts)
+    private fun searchItemsForStore(storeUrl: String, brand: String?, article: String, pages: Int, products: MutableCollection<Product>) {
+        val productName = if (article == articles[0]) "Cafetera" else article
+        val result = when (storeUrl) {
+            CONSTANTS.FNAC.URL -> fnacSearchProduct(productName, brand, pages)
+            CONSTANTS.AMAZON.URL -> amazonSearchProduct(productName, brand, pages)
+            CONSTANTS.CORTEINGLES.URL -> corteInglesSearchProduct(productName, brand, pages)
+            else -> emptyList()
         }
+        addProcessedProducts(result)
+        products.addAll(result)
     }
 
+
 }
diff --git a/src/main/kotlin/datasource/Stores/StoreRepository.kt b/src/main/kotlin/datasource/Stores/StoreRepository.kt
@@ -5,11 +5,12 @@ import org.openqa.selenium.By
 import org.openqa.selenium.JavascriptExecutor
 import org.openqa.selenium.WebDriver
 import org.openqa.selenium.WebElement
+import org.openqa.selenium.chrome.ChromeDriver
 import org.openqa.selenium.interactions.Actions
 
 
 abstract class StoreRepository {
-    val driver: WebDriver by kdi()
+    val driver: WebDriver = ChromeDriver()
     abstract fun waitForPageToLoad()
 
     fun browse(str: String) {
@@ -39,4 +40,8 @@ abstract class StoreRepository {
         (driver as JavascriptExecutor).executeScript("arguments[0].scrollIntoView(true);", element)
         Thread.sleep(1000)
     }
+
+    fun destroy() {
+        driver.close()
+    }
 }
diff --git a/src/main/kotlin/usecases/AmazonSearchProduct.kt b/src/main/kotlin/usecases/AmazonSearchProduct.kt
@@ -1,6 +1,6 @@
 package usecases
 
-import datasource.BrandDAO
+import datasource.Stores.AmazonRepository
 import datasource.Stores.StoreRepository
 import di.kdi
 import domain.BrandNotFoundException
@@ -11,7 +11,7 @@ import org.openqa.selenium.WebElement
 
 class AmazonSearchProduct : ISearchProducts() {
 
-    override val webDriver: StoreRepository by kdi(CONSTANTS.AMAZON.URL)
+    override val webDriver: StoreRepository = AmazonRepository()
 
     override operator fun invoke(productName: String, brand: String?, page: Int): List<Product> {
         val result = mutableListOf<Product>()
@@ -21,12 +21,14 @@ class AmazonSearchProduct : ISearchProducts() {
 
         webDriver.search(inputBar, productName)
         webDriver.waitForPageToLoad()
+        Thread.sleep(2000)
 
         if (brand != null) {
             try {
                 this.selectBrand(brand)
                 Thread.sleep(1000)
             } catch (e: BrandNotFoundException) {
+                webDriver.destroy()
                 return result
             }
         }
@@ -45,11 +47,13 @@ class AmazonSearchProduct : ISearchProducts() {
             try {
                 val nextPage = webDriver.findElement(By.id("pagnNextLink")).getAttribute("href")
                 webDriver.browse(nextPage)
-            } catch (ex: Exception) {
+            } catch (e: Exception) {
+                webDriver.destroy()
                 return result
             }
         }
 
+        webDriver.destroy()
         return result
     }
 

diff --git a/src/main/kotlin/usecases/ElCorteInglesSearchProduct.kt b/src/main/kotlin/usecases/ElCorteInglesSearchProduct.kt
@@ -1,6 +1,7 @@
 package usecases
 
 import datasource.BrandDAO
+import datasource.Stores.CorteInglesRepository
 import datasource.Stores.StoreRepository
 import di.kdi
 import domain.BrandNotFoundException
@@ -10,7 +11,7 @@ import org.openqa.selenium.By
 import org.openqa.selenium.WebElement
 
 class ElCorteInglesSearchProduct: ISearchProducts() {
-    override val webDriver: StoreRepository by kdi(CONSTANTS.CORTEINGLES.URL)
+    override val webDriver: StoreRepository = CorteInglesRepository()
 
     override operator fun invoke(productName: String, brand: String?, page: Int): List<Product> {
         val result = mutableListOf<Product>()
@@ -26,6 +27,7 @@ class ElCorteInglesSearchProduct: ISearchProducts() {
                 this.selectBrand(brand)
                 Thread.sleep(1000)
             } catch (e: BrandNotFoundException) {
+                webDriver.destroy()
                 return result
             }
         }
@@ -50,10 +52,12 @@ class ElCorteInglesSearchProduct: ISearchProducts() {
                 val nextPage = webDriver.findElement(By.cssSelector("ul.a-unordered-list::nth-child(8)")).getAttribute("href")
                 webDriver.browse(nextPage)
             } catch (ex: Exception) {
+                webDriver.destroy()
                 return result
             }
         }
 
+        webDriver.destroy()
         return result
     }
 

diff --git a/src/main/kotlin/usecases/FnacSearchProduct.kt b/src/main/kotlin/usecases/FnacSearchProduct.kt
@@ -1,5 +1,6 @@
 package usecases
 
+import datasource.Stores.FnacRepository
 import datasource.BrandDAO
 import datasource.Stores.StoreRepository
 import di.kdi
@@ -11,7 +12,7 @@ import org.openqa.selenium.WebElement
 
 class FnacSearchProduct : ISearchProducts() {
 
-    override val webDriver: StoreRepository by kdi(CONSTANTS.FNAC.URL)
+    override val webDriver: StoreRepository = FnacRepository()
     private val allBrandsList: List<String> = brandsDAO.getAll()
 
     override fun invoke(productName: String, brand: String?, page: Int): List<Product> {
@@ -36,6 +37,7 @@ class FnacSearchProduct : ISearchProducts() {
                 this.selectBrand(brand)
                 Thread.sleep(1000)
             } catch (e: BrandNotFoundException) {
+                webDriver.destroy()
                 return result
             }
         }
@@ -53,10 +55,12 @@ class FnacSearchProduct : ISearchProducts() {
             try {
                 webDriver.findElement(By.className("actionNext")).click()
             } catch (ex: Exception) {
+                webDriver.destroy()
                 return result
             }
         }
 
+        webDriver.destroy()
         return result
 
     }

diff --git a/src/main/kotlin/usecases/ISearchProducts.kt b/src/main/kotlin/usecases/ISearchProducts.kt
@@ -17,7 +17,7 @@ abstract class ISearchProducts {
                 "de", "para", "capsulas", "espresso", "automatica", "filtro", "molinillo", "multibebida",
                 "electrica", "expresso", "vivy", "grind", "goteo", "brew", "superautomatica", "n",
                 "senseo", "intense", "coffee", "maker", "blanco", "blanca", "con", "jarra",
-                "dolce", "gusto", "f", "nescafe", "italiana", "hidro-presion", "russell", "hobbs"
+                "dolce", "gusto", "f", "nescafe", "italiana", "hidro-presion", "russell", "hobbs", "super"
         ).union(brandsDAO.getAll().map(String::toLowerCase)).toList()
     }
 

diff --git a/src/main/resources/productos.txt b/src/main/resources/productos.txt
@@ -1,6 +1,6 @@
-Cafeteras italianas
-Cafeteras de goteo
-Cafeteras de espresso manuales
-Cafeteras de capsulas
-Cafeteras automaticas
-Cafeteras combinadas espresso/goteo
+Cafetera italianas
+Cafetera de goteo
+Cafetera espresso manual
+Cafetera capsulas
+Cafetera automatica
+Cafetera combinada espresso y goteo