From 93c0c49e9aba719c8c8e6b171e0dd515a586d32b Mon Sep 17 00:00:00 2001 From: Noemi Vanyi Date: Sun, 17 Jul 2016 18:42:30 +0200 Subject: [PATCH 01/14] add time range search with yahoo --- searx/engines/__init__.py | 3 ++- searx/engines/yahoo.py | 33 +++++++++++++++++++++++++-------- searx/search.py | 9 +++++++-- searx/webapp.py | 1 + 4 files changed, 35 insertions(+), 11 deletions(-) diff --git a/searx/engines/__init__.py b/searx/engines/__init__.py index 099baa587..2c735a188 100644 --- a/searx/engines/__init__.py +++ b/searx/engines/__init__.py @@ -42,7 +42,8 @@ engine_default_args = {'paging': False, 'shortcut': '-', 'disabled': False, 'suspend_end_time': 0, - 'continuous_errors': 0} + 'continuous_errors': 0, + 'time_range_support': False} def load_module(filename): diff --git a/searx/engines/yahoo.py b/searx/engines/yahoo.py index b8b40e4aa..2334614cb 100644 --- a/searx/engines/yahoo.py +++ b/searx/engines/yahoo.py @@ -20,10 +20,12 @@ from searx.engines.xpath import extract_text, extract_url categories = ['general'] paging = True language_support = True +time_range_support = True # search-url base_url = 'https://search.yahoo.com/' search_url = 'search?{query}&b={offset}&fl=1&vl=lang_{lang}' +search_url_with_time = 'search?{query}&b={offset}&fl=1&vl=lang_{lang}&age={age}&btf={btf}&fr2=time' # specific xpath variables results_xpath = "//div[contains(concat(' ', normalize-space(@class), ' '), ' Sr ')]" @@ -32,6 +34,9 @@ title_xpath = './/h3/a' content_xpath = './/div[@class="compText aAbs"]' suggestion_xpath = "//div[contains(concat(' ', normalize-space(@class), ' '), ' AlsoTry ')]//a" +time_range_dict = {'day': ['1d', 'd'], + 'week': ['1w', 'w'], + 'month': ['1m', 'm']} # remove yahoo-specific tracking-url def parse_url(url_string): @@ -51,18 +56,30 @@ def parse_url(url_string): return unquote(url_string[start:end]) +def _get_url(query, offset, language, time_range): + if time_range: + return base_url + search_url_with_time.format(offset=offset, + query=urlencode({'p': query}), + lang=language, + age=time_range_dict[time_range][0], + btf=time_range_dict[time_range][1]) + return base_url + search_url.format(offset=offset, + query=urlencode({'p': query}), + lang=language) + + +def _get_language(params): + if params['language'] == 'all': + return 'en' + return params['language'].split('_')[0] + + # do search-request def request(query, params): offset = (params['pageno'] - 1) * 10 + 1 + language = _get_language(params) - if params['language'] == 'all': - language = 'en' - else: - language = params['language'].split('_')[0] - - params['url'] = base_url + search_url.format(offset=offset, - query=urlencode({'p': query}), - lang=language) + params['url'] = _get_url(query, offset, language, params['time_range']) # TODO required? params['cookies']['sB'] = 'fl=1&vl=lang_{lang}&sh=1&rw=new&v=1'\ diff --git a/searx/search.py b/searx/search.py index a40801640..377d9ceda 100644 --- a/searx/search.py +++ b/searx/search.py @@ -138,6 +138,7 @@ class Search(object): self.paging = False self.pageno = 1 self.lang = 'all' + self.time_range = None # set blocked engines self.disabled_engines = request.preferences.engines.get_disabled() @@ -178,9 +179,9 @@ class Search(object): if len(query_obj.languages): self.lang = query_obj.languages[-1] - self.engines = query_obj.engines + self.time_range = self.request_data.get('time_range') - self.categories = [] + self.engines = query_obj.engines # if engines are calculated from query, # set categories by using that informations @@ -279,6 +280,9 @@ class Search(object): if self.lang != 'all' and not engine.language_support: continue + if self.time_range and not engine.time_range_support: + continue + # set default request parameters request_params = default_request_params() request_params['headers']['User-Agent'] = user_agent @@ -293,6 +297,7 @@ class Search(object): # 0 = None, 1 = Moderate, 2 = Strict request_params['safesearch'] = request.preferences.get_value('safesearch') + request_params['time_range'] = self.time_range # update request parameters dependent on # search-engine (contained in engines folder) diff --git a/searx/webapp.py b/searx/webapp.py index e9d27a0db..7ae826026 100644 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -459,6 +459,7 @@ def index(): paging=search.paging, number_of_results=format_decimal(number_of_results), pageno=search.pageno, + time_range=search.time_range, base_url=get_base_url(), suggestions=search.result_container.suggestions, answers=search.result_container.answers, From 2e8ab34d76fa0828199697d1f8fe2b37b4dc29b9 Mon Sep 17 00:00:00 2001 From: Noemi Vanyi Date: Sun, 17 Jul 2016 18:43:47 +0200 Subject: [PATCH 02/14] add advanced settings for logicodev style --- .../static/themes/oscar/css/logicodev.min.css | Bin 6985 -> 8770 bytes .../themes/oscar/less/logicodev/advanced.less | 70 ++++++++++++++++++ .../themes/oscar/less/logicodev/oscar.less | 2 + .../themes/oscar/less/logicodev/search.less | 44 +++++------ .../oscar/less/logicodev/variables.less | 2 + searx/templates/oscar/advanced.html | 11 +++ searx/templates/oscar/categories.html | 48 +++--------- searx/templates/oscar/preferences.html | 2 +- searx/templates/oscar/results.html | 9 ++- searx/templates/oscar/search.html | 4 +- searx/templates/oscar/search_full.html | 9 +-- searx/templates/oscar/time-range.html | 11 +++ 12 files changed, 136 insertions(+), 76 deletions(-) create mode 100644 searx/static/themes/oscar/less/logicodev/advanced.less create mode 100644 searx/templates/oscar/advanced.html create mode 100644 searx/templates/oscar/time-range.html diff --git a/searx/static/themes/oscar/css/logicodev.min.css b/searx/static/themes/oscar/css/logicodev.min.css index aab5fb8415c145611ae4c8e41ebaa4fbc4127f9f..990ee2fe240cafd41f47063af9d593b95d50cfdd 100644 GIT binary patch delta 2155 zcmcIm%WmT~6vY7TX3$_deW2)~fB~aG2Z<8-5hsz+1?5PN6+ac(iB)t#iL@;1L0O_B z+i9wQXjc7;0<-A8yZ%)vIZ5p?#w{?d0aL`eyqA|p=U(d1${**y)%G}w=ai!%^rI08 zJ;8*5O`&Tqan*yNW&4~#C?%e7cpPHyA=ul0aquwR`#d?c{a{8pDj#{0ZYjCt>@g!>2^;bC4q3bfw*`?R|kAvw2J$5 z=&#=hE{rH=j%^ohoYRE{cGrz5Ip+51*O%<(X>H}*#vwNuV6WnfhhIPX=Cdq=jhs^W z;|zZI4GUaqd%S`#8Kqb`-d9A!M{> z0%3}pmTbvvSHK9n*razb2$=R#6|S_# z#n^Cc%5C8uS#E)w{UrIohVp^kqmZn(m{s>kJLXflto4(%J}1mV_d5hB6l@P#_$cr# zW{q&Ad|(g6jvDiqb&CM@YhU%pQr}9ar|HnGfgXF*)KZ}&ra&Urk*M`00Gjn~?Pxw0 zGrKssl9e_9Vnt!jR)c}vG++%>*oZVXbb>T&#bbk9s+lua%60&r(p32#I4a}rvYA1O zDUEeAX3=DatyXm~@C~J(&Ovg4B)6^hMwU3Yn&+`tEh68H0&N}TCTq=&Fp>L-8Y=$nU5*fTDk%&9c9*- z-VT>n3Yo@Jae8VYPt%g?)5;OD#&gi0mgmu6sGYe`9NyI8vl4)l@kzZa$;A>aj~sD` zZ|kxK+D=zZ6I561Qqm+?udw z_e4Fn(g;9D(qM?>_O0%#rQwa@f(ukai{rrX=yF(;WBuy**7UG(GKpd803=t|oU2*S ztJU7*&ML{bIiGMLW(7s>^I3VaYkAe-->LIW<5MaB{QiFQ>~FZlbnXwI<34^Oq&Mo9 T=}k~wyDT!rzy0OY-(UU(Ey@0R delta 440 zcmaKp(MrQG6o&CuC|*n$f)|D2m1d+{RZK7EjhDWFaMEO5u+52SoYmqRc-I#Z273VS zd>&uG)NZiBkidcbf6jl-N4{>quRf;%DZ%DSvzVGXq+!z4)XCZrXF*ppv?a^HC7q+KVUyCv&t)R$&2L7QNoABgQwAI1wp_>@m0atOe + + +
+ {% include 'oscar/categories.html' %} + {% include 'oscar/time-range.html' %} +
+ diff --git a/searx/templates/oscar/categories.html b/searx/templates/oscar/categories.html index 834cffcce..d850bb744 100644 --- a/searx/templates/oscar/categories.html +++ b/searx/templates/oscar/categories.html @@ -1,42 +1,14 @@ - -