From 90e1db3e5c628ae73800d51b53032064aadbfb44 Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Sat, 31 Dec 2016 13:56:09 +0100 Subject: [PATCH 1/8] [fix] extract_text: use html.tostring instead html_to_text. Fix #711 --- searx/engines/xpath.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/searx/engines/xpath.py b/searx/engines/xpath.py index e5c0c5bea..18943bba4 100644 --- a/searx/engines/xpath.py +++ b/searx/engines/xpath.py @@ -42,7 +42,9 @@ def extract_text(xpath_results): return ''.join(xpath_results) else: # it's a element - return html_to_text(xpath_results.text_content()).strip() + text = html.tostring(xpath_results, encoding='unicode', method='text', with_tail=False) + text = text.strip().replace('\n', ' ') + return ' '.join(text.split()) def extract_url(xpath_results, search_url): From 20c94592e72d90856dcc023a3345dbccde60d5bc Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Mon, 2 Jan 2017 14:11:48 +0100 Subject: [PATCH 2/8] [mod] upgrade versions in requirements.txt --- requirements.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/requirements.txt b/requirements.txt index c4cbe4e04..d6c91b246 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,12 @@ certifi==2016.9.26 -flask==0.11.1 +flask==0.12 flask-babel==0.11.1 -lxml==3.6.0 -ndg-httpsclient==0.4.1 +lxml==3.7.1 +ndg-httpsclient==0.4.2 pyasn1==0.1.9 pyasn1-modules==0.0.8 pygments==2.1.3 -pyopenssl==0.15.1 +pyopenssl==16.2.0 python-dateutil==2.5.3 pyyaml==3.11 -requests[socks]==2.10.0 +requests[socks]==2.12.4 From 7b1daf254ef71cd498c33a002130b0ebc8394195 Mon Sep 17 00:00:00 2001 From: Adam Tauber Date: Tue, 3 Jan 2017 13:11:38 +0100 Subject: [PATCH 3/8] [fix] autocomplete unicode issue - closes #808 --- searx/autocomplete.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/searx/autocomplete.py b/searx/autocomplete.py index dff8a69a0..b360af9f6 100644 --- a/searx/autocomplete.py +++ b/searx/autocomplete.py @@ -81,22 +81,22 @@ def searx_bang(full_query): engine_query = full_query.getSearchQuery()[1:] for lc in language_codes: - lang_id, lang_name, country, english_name = map(str.lower, lc) + lang_id, lang_name, country, english_name = map(unicode.lower, lc) # check if query starts with language-id if lang_id.startswith(engine_query): if len(engine_query) <= 2: - results.append(':{lang_id}'.format(lang_id=lang_id.split('-')[0])) + results.append(u':{lang_id}'.format(lang_id=lang_id.split('-')[0])) else: - results.append(':{lang_id}'.format(lang_id=lang_id)) + results.append(u':{lang_id}'.format(lang_id=lang_id)) # check if query starts with language name if lang_name.startswith(engine_query) or english_name.startswith(engine_query): - results.append(':{lang_name}'.format(lang_name=lang_name)) + results.append(u':{lang_name}'.format(lang_name=lang_name)) # check if query starts with country if country.startswith(engine_query.replace('_', ' ')): - results.append(':{country}'.format(country=country.replace(' ', '_'))) + results.append(u':{country}'.format(country=country.replace(' ', '_'))) # remove duplicates result_set = set(results) From 7bcdc27a2e2bf7700cd0b1515bcd5e41b40d0bea Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Tue, 3 Jan 2017 19:15:55 +0100 Subject: [PATCH 4/8] [fix] fix standalone_search.py --- utils/standalone_search.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/utils/standalone_search.py b/utils/standalone_search.py index 7e9516f82..cad32eeca 100644 --- a/utils/standalone_search.py +++ b/utils/standalone_search.py @@ -13,10 +13,9 @@ request_params = default_request_params() # Possible params # request_params['headers']['User-Agent'] = '' # request_params['category'] = '' -# request_params['started'] = '' - request_params['pageno'] = 1 request_params['language'] = 'en_us' +request_params['time_range'] = '' params = google.request(argv[1], request_params) @@ -32,5 +31,5 @@ else: request_args['data'] = request_params['data'] resp = req(request_params['url'], **request_args) - +resp.search_params = request_params print(dumps(google.response(resp))) From 6db25fc5c2d2459c24df4a29991484b285aba08e Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Tue, 3 Jan 2017 19:25:04 +0100 Subject: [PATCH 5/8] [mod] ./manage.sh can download geckodriver and install it into the virtual environment --- manage.sh | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/manage.sh b/manage.sh index 11f2df04d..a361c5616 100755 --- a/manage.sh +++ b/manage.sh @@ -14,6 +14,36 @@ update_dev_packages() { pip install --upgrade -r "$BASE_DIR/requirements-dev.txt" } +check_geckodriver() { + echo '[!] Checking geckodriver' + set -e + geckodriver -V 2>1 > /dev/null || NOTFOUND=1 + set +e + if [ -z $NOTFOUND ]; then + return + fi + GECKODRIVER_VERSION="v0.11.1" + PLATFORM=`python -c "import platform; print platform.system().lower(), platform.architecture()[0]"` + case $PLATFORM in + "linux 32bit" | "linux2 32bit") ARCH="linux32";; + "linux 64bit" | "linux2 64bit") ARCH="linux64";; + "windows 32 bit") ARCH="win32";; + "windows 64 bit") ARCH="win64";; + "mac 64bit") ARCH="macos";; + esac + GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-$ARCH.tar.gz"; + if [ -z "$VIRTUAL_ENV" ]; then + echo "geckodriver can't be installed because VIRTUAL_ENV is not set, you should download it from\n $GECKODRIVER_URL" + exit + else + echo "Installing $VIRTUAL_ENV from\n $GECKODRIVER_URL" + FILE=`mktemp` + wget "$GECKODRIVER_URL" -qO $FILE && tar xz -C $VIRTUAL_ENV/bin/ -f $FILE geckodriver + rm $FILE + chmod 777 $VIRTUAL_ENV/bin/geckodriver + fi +} + pep8_check() { echo '[!] Running pep8 check' # ignored rules: @@ -43,6 +73,7 @@ tests() { set -e pep8_check unit_tests + check_geckodriver robot_tests set +e } @@ -88,6 +119,7 @@ Commands unit_tests - Run unit tests update_dev_packages - Check & update development and production dependency changes update_packages - Check & update dependency changes + check_geckodriver - Check & download geckodriver (required for robot_tests) " } From 8af29d9797ce0b8ba06c1440d4d074c487994e1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Tue, 3 Jan 2017 22:03:59 +0100 Subject: [PATCH 6/8] Revert "Merge pull request #799 from kvch/modal-arrows" This reverts commit 79ceb13577fcc83eabe642a4ed6dd793b7bf0d4b, reversing changes made to eaa0fb8102825fa4ccdb348429fdbf39837921be. --- .../static/themes/oscar/css/logicodev.min.css | Bin 12089 -> 11209 bytes searx/static/themes/oscar/css/pointhi.min.css | Bin 8317 -> 7437 bytes .../oscar/less/logicodev/modal-pic.less | 24 +++--------------- .../oscar/result_templates/images.html | 4 +-- 4 files changed, 6 insertions(+), 22 deletions(-) diff --git a/searx/static/themes/oscar/css/logicodev.min.css b/searx/static/themes/oscar/css/logicodev.min.css index 46b8b98adbb3e7b5acf4d06e5ae562f6a8ccde7c..4f5acfff3176d0c2a07a98ae85af14d0e468e15a 100644 GIT binary patch delta 88 zcmdlPcQSlKih($ delta 846 zcmc(dO>Ub|6opsN4YZR?I%uo}1I7-HGI+ovfDPsW4{Q%AVe@PAXAA)&Rhe`LSx7tW z61sM5UnJewUKOjj&`5d zMlM57v25y07HrNPw+BU7Fxxre{NpDrtnNCJNR)j3&ZNL$G6`C_*^fU_P)0jKc z$EmhthbsW57?^?OQ3=EF;o-v-JAHqZp>>T8cJd@0-}V}# z44m8_v%R#s^WERMMsLV=PqUcVadyjrFv1v=01ZZ1W{W4 zYUVjZu`rgppmar)%!^=u!03|VFTYOwHtLM>&B896i`5k3lyJ2)!nMGkIkGeo89It( z^ZQwZr6v%-5F#5Tb$@APs4XBvc<7q$5lM27_8~aP&@Hu@X=5PN9T2 zjV=mBfVyjMCE|WnR+dJ*rKq#cyx?M;^JJ8?>s=_g?fh>J&iX&# zAZ{teN&hPj&isoUko_|U1I{|4j)*}uX`AW-OkiC;Y{|l>DLjF2gI0
- + - +

{{ result.title|striptags }}

From e8319fa2cbd388712a39202d44c1b6a3a520b048 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Tue, 3 Jan 2017 22:04:09 +0100 Subject: [PATCH 7/8] Revert "Merge pull request #793 from kvch/pics-modal" This reverts commit bff41987a49af9ae53ff9fc3494fa7973ac03500, reversing changes made to d08108be6293c8764f0d1d950a7aab283cddffed. --- searx/static/plugins/js/infinite_scroll.js | 4 - .../static/themes/oscar/css/logicodev.min.css | Bin 11209 -> 10469 bytes searx/static/themes/oscar/css/pointhi.min.css | Bin 7437 -> 6697 bytes .../oscar/less/logicodev/modal-pic.less | 61 --------------- .../themes/oscar/less/logicodev/oscar.less | 2 - .../themes/oscar/less/pointhi/oscar.less | 4 - .../oscar/result_templates/images.html | 73 +++++++----------- 7 files changed, 29 insertions(+), 115 deletions(-) delete mode 100644 searx/static/themes/oscar/less/logicodev/modal-pic.less diff --git a/searx/static/plugins/js/infinite_scroll.js b/searx/static/plugins/js/infinite_scroll.js index 1bc27e2e7..9cd582d7f 100644 --- a/searx/static/plugins/js/infinite_scroll.js +++ b/searx/static/plugins/js/infinite_scroll.js @@ -4,13 +4,9 @@ $(document).ready(function() { if ($(document).height() - win.height() == win.scrollTop()) { var formData = $('#pagination form:last').serialize(); if (formData) { - var pageno = $('#pagination input[name=pageno]:last').attr('value'); $('#pagination').html(''); $.post('./', formData, function (data) { - var lastImageHref = $('.result-images:last a').attr('href'); var body = $(data); - $('a[href^="#open-modal"]:last').attr('href', '#open-modal-1-' + pageno); - body.find('.modal-image a:first').attr('href', lastImageHref); $('#pagination').remove(); $('#main_results').append('
'); $('#main_results').append(body.find('.result')); diff --git a/searx/static/themes/oscar/css/logicodev.min.css b/searx/static/themes/oscar/css/logicodev.min.css index 4f5acfff3176d0c2a07a98ae85af14d0e468e15a..33d451b8f1b712e5bd5d8f106b4d1deb13fa6da8 100644 GIT binary patch delta 7 OcmX>Z{xopIQw;zSPXrVI delta 753 zcmaKqO>ToQ5QVQo6Gcj-f*`Gu#+IsHp{uS9He^)nv5b?FAjFk=jUFWTsDb>DwkckK z@%Wp0Gw<;F_4av~Hi1OWavvniM5VC+3#dp6rQp<}CLuMDt*v{ zi#IN%syGxnXS?44#4?i!Tb=095BdPkHWsho&m6XuY#aa-nNPenfPMhCYqDBSXnavO zP8aDaWv&5+3q!{2v0Yve86mv_bKv!h6dag0m7x5dv0SWlIje?p`i!hRMl3>izp1Z^ z@`7#ToQ5QXp26-^W=kqUw|N*Y_LdKH5W83lVRcFGND8Ik)S@OKHIQwrd&g)kO81eAwA-^}n;G0G zPD~cFY53Hx;AKXSJ}53q5{0KHqNrH{_CUf%kQ$9;%2JJ>8Y_%1_Cnc$C{&^z*5k!H zmr_+63Z1jV?*L+%$%L)W^ymkD0B1XkH}F>u`${$r0E)~fUK>E)f!hsPuO>9UsvD=v zbe%HS0KlbhoWFE}j6{;EM?P))lwu=84UDRDj)8!yhNB$F&xbzH^6Ha20 zq7j#I#5pEgdX?N(LglT#?sy|Q3q`G9dfGg9o5Y#Ty5W=2=$t&svAE;JlkqONa59*8 P-*#uY;m@-Z{iyy1yfG0_ diff --git a/searx/static/themes/oscar/less/logicodev/modal-pic.less b/searx/static/themes/oscar/less/logicodev/modal-pic.less deleted file mode 100644 index 6861f1710..000000000 --- a/searx/static/themes/oscar/less/logicodev/modal-pic.less +++ /dev/null @@ -1,61 +0,0 @@ -.modal-image { - position: fixed; - top: 0; - right: 0; - bottom: 0; - left: 0; - background: rgba(0,0,0,0.8); - z-index: 1000000001; - opacity:0 !important; - pointer-events: none; - - button { - display: none; - } - - &:target { - opacity: 1 !important; - pointer-events: auto; - } - - & > div { - margin: 2% auto; - width: 97%; - background: @dim-gray; - border: @gray 0.1rem solid; - } - - @media (min-width: 769px) { - & > div { - max-width: 60.0rem; - } - } - - .image-paging-left { - font-size: 2.2rem; - padding-right: 1.0rem; - } - - .image-paging-right { - font-size: 2.2rem; - padding-left: 1.2rem; - } - - .image-container::before { - display: block; - min-width: 1.0rem; - max-width: 60.0rem; - min-height: 10.0rem; - height: 30.0rem; - content: ""; - } -} - -.modal-close { - position:fixed; - top: 0; - left: 0; - height: 100% !important; - width: 100% !important; - z-index: -1; -} diff --git a/searx/static/themes/oscar/less/logicodev/oscar.less b/searx/static/themes/oscar/less/logicodev/oscar.less index ecb3a0799..55181cb9c 100644 --- a/searx/static/themes/oscar/less/logicodev/oscar.less +++ b/searx/static/themes/oscar/less/logicodev/oscar.less @@ -19,5 +19,3 @@ @import "cursor.less"; @import "code.less"; - -@import "modal-pic.less"; diff --git a/searx/static/themes/oscar/less/pointhi/oscar.less b/searx/static/themes/oscar/less/pointhi/oscar.less index 4529801bb..4e2fee129 100644 --- a/searx/static/themes/oscar/less/pointhi/oscar.less +++ b/searx/static/themes/oscar/less/pointhi/oscar.less @@ -17,7 +17,3 @@ @import "code.less"; @import "navbar.less"; - -@import "../logicodev/variables.less"; - -@import "../logicodev/modal-pic.less"; diff --git a/searx/templates/oscar/result_templates/images.html b/searx/templates/oscar/result_templates/images.html index 3bf52cd68..b23f34915 100644 --- a/searx/templates/oscar/result_templates/images.html +++ b/searx/templates/oscar/result_templates/images.html @@ -1,54 +1,39 @@ - - {{ result.title|striptags }} +{% from 'oscar/macros.html' import draw_favicon %} + + + {{ result.title|striptags }} - - -