Auto merge of #1995 - Valloric:ycmd-update, r=vheon

[READY] Updating to latest ycmd

Still using ycmd only in py2, but at least we can use YCM with ycmd master again.

@micbou @puremourning @vheon Try to give this one a whirl on your machines since ycmd had a ton of recent changes.

<!-- Reviewable:start -->
[<img src="https://reviewable.io/review_button.svg" height="40" alt="Review on Reviewable"/>](https://reviewable.io/reviews/valloric/youcompleteme/1995)
<!-- Reviewable:end -->

autoload/youcompleteme.vim

@@ -139,12 +139,13 @@ import subprocess
 script_folder = vim.eval( 's:script_folder_path' )
 sys.path.insert( 0, os.path.join( script_folder, '../python' ) )
 sys.path.insert( 0, os.path.join( script_folder, '../third_party/ycmd' ) )
-from ycmd import utils
-utils.AddNearestThirdPartyFoldersToSysPath( script_folder )
+from ycmd import server_utils
+server_utils.AddNearestThirdPartyFoldersToSysPath( script_folder )
 
 # We need to import ycmd's third_party folders as well since we import and
 # use ycmd code in the client.
-utils.AddNearestThirdPartyFoldersToSysPath( utils.__file__ )
+server_utils.AddNearestThirdPartyFoldersToSysPath( server_utils.__file__ )
+from ycmd import utils
 from ycm import base
 base.LoadJsonDefaultsIntoVim()
 from ycmd import user_options_store

install.py

@@ -6,6 +6,11 @@ import sys
 import os.path as p
 import glob
 
+major, minor = sys.version_info[ 0 : 2 ]
+if major != 2 or minor < 6:
+  sys.exit( 'The build script requires Python version >= 2.6 and < 3.0; '
+            'your version of Python is ' + sys.version )
+
 DIR_OF_THIS_SCRIPT = p.dirname( p.abspath( __file__ ) )
 DIR_OF_OLD_LIBS = p.join( DIR_OF_THIS_SCRIPT, 'python' )
 

python/test_requirements.txt

@@ -2,3 +2,5 @@ flake8>=2.0
 mock>=1.0.1
 nose>=1.3.0
 PyHamcrest>=1.8.0
+# This needs to be kept in sync with submodule checkout in third_party/ycmd/third_party
+future==0.15.2

python/ycm/client/base_request.py

@@ -17,13 +17,14 @@
 
 import requests
 import urlparse
+import json
 from base64 import b64decode, b64encode
 from retries import retries
 from requests_futures.sessions import FuturesSession
 from ycm.unsafe_thread_pool_executor import UnsafeThreadPoolExecutor
 from ycm import vimsupport
-from ycmd.utils import ToUtf8Json
-from ycmd.hmac_utils import CreateRequestHmac, CreateHmac, SecureStringsEqual
+from ycmd.utils import ToBytes
+from ycmd.hmac_utils import CreateRequestHmac, CreateHmac, SecureBytesEqual
 from ycmd.responses import ServerError, UnknownExtraConf
 
 _HEADERS = {'content-type': 'application/json'}
@@ -89,7 +90,7 @@ class BaseRequest( object ):
     def SendRequest( data, handler, method, timeout ):
       request_uri = _BuildUri( handler )
       if method == 'POST':
-        sent_data = ToUtf8Json( data )
+        sent_data = _ToUtf8Json( data )
         return BaseRequest.session.post(
             request_uri,
             data = sent_data,
@@ -107,7 +108,7 @@ class BaseRequest( object ):
     def DelayedSendRequest( data, handler, method ):
       request_uri = _BuildUri( handler )
       if method == 'POST':
-        sent_data = ToUtf8Json( data )
+        sent_data = _ToUtf8Json( data )
         return requests.post(
             request_uri,
             data = sent_data,
@@ -182,13 +183,18 @@ def HandleServerException( exception ):
   vimsupport.PostVimMessage( serialized_exception )
 
 
+def _ToUtf8Json( data ):
+  return ToBytes( json.dumps( data ) if data else None )
+
+
 def _ValidateResponseObject( response ):
-  hmac = CreateHmac( response.content, BaseRequest.hmac_secret )
-  if not SecureStringsEqual( hmac,
-                             b64decode( response.headers[ _HMAC_HEADER ] ) ):
+  our_hmac = CreateHmac( response.content, BaseRequest.hmac_secret )
+  their_hmac = ToBytes( b64decode( response.headers[ _HMAC_HEADER ] ) )
+  if not SecureBytesEqual( our_hmac, their_hmac ):
     raise RuntimeError( 'Received invalid HMAC for response!' )
   return True
 
+
 def _BuildUri( handler ):
   return urlparse.urljoin( BaseRequest.server_location, handler )
 
@@ -216,6 +222,7 @@ def _CheckServerIsHealthyWithCache():
   except:
     return False
 
+
 def MakeServerException( data ):
   if data[ 'exception' ][ 'TYPE' ] == UnknownExtraConf.__name__:
     return UnknownExtraConf( data[ 'exception' ][ 'extra_conf_file' ] )

python/ycm/client/command_request.py

@@ -18,7 +18,7 @@
 import vim
 from ycm.client.base_request import BaseRequest, BuildRequestData, ServerError
 from ycm import vimsupport
-from ycmd.utils import ToUtf8IfNeeded
+from ycmd.utils import ToUnicode
 
 
 def _EnsureBackwardsCompatibility( arguments ):
@@ -123,9 +123,9 @@ def SendCommandRequest( arguments, completer ):
 def _BuildQfListItem( goto_data_item ):
   qf_item = {}
   if 'filepath' in goto_data_item:
-    qf_item[ 'filename' ] = ToUtf8IfNeeded( goto_data_item[ 'filepath' ] )
+    qf_item[ 'filename' ] = ToUnicode( goto_data_item[ 'filepath' ] )
   if 'description' in goto_data_item:
-    qf_item[ 'text' ] = ToUtf8IfNeeded( goto_data_item[ 'description' ] )
+    qf_item[ 'text' ] = ToUnicode( goto_data_item[ 'description' ] )
   if 'line_num' in goto_data_item:
     qf_item[ 'lnum' ] = goto_data_item[ 'line_num' ]
   if 'column_num' in goto_data_item:

python/ycm/client/completion_request.py

@@ -15,7 +15,7 @@
 # You should have received a copy of the GNU General Public License
 # along with YouCompleteMe.  If not, see <http://www.gnu.org/licenses/>.
 
-from ycmd.utils import ToUtf8IfNeeded
+from ycmd.utils import ToBytes, ToUnicode
 from ycm.client.base_request import ( BaseRequest, JsonFromFuture,
                                       HandleServerException,
                                       MakeServerException )
@@ -69,22 +69,22 @@ def ConvertCompletionDataToVimData( completion_data ):
 
   if ( 'extra_data' in completion_data and
        'doc_string' in completion_data[ 'extra_data' ] ):
-    doc_string = ToUtf8IfNeeded(
-                              completion_data[ 'extra_data' ][ 'doc_string' ] )
+    doc_string = ToBytes( completion_data[ 'extra_data' ][ 'doc_string' ] )
   else:
     doc_string = ""
 
   if 'insertion_text' in completion_data:
-    vim_data[ 'word' ] = ToUtf8IfNeeded( completion_data[ 'insertion_text' ] )
+    vim_data[ 'word' ] = ToBytes( completion_data[ 'insertion_text' ] )
   if 'menu_text' in completion_data:
-    vim_data[ 'abbr' ] = ToUtf8IfNeeded( completion_data[ 'menu_text' ] )
+    vim_data[ 'abbr' ] = ToBytes( completion_data[ 'menu_text' ] )
   if 'extra_menu_info' in completion_data:
-    vim_data[ 'menu' ] = ToUtf8IfNeeded( completion_data[ 'extra_menu_info' ] )
+    vim_data[ 'menu' ] = ToBytes( completion_data[ 'extra_menu_info' ] )
   if 'kind' in completion_data:
-    vim_data[ 'kind' ] = ToUtf8IfNeeded(
-        completion_data[ 'kind' ] )[ 0 ].lower()
+    kind = ToUnicode( completion_data[ 'kind' ] )
+    if kind:
+      vim_data[ 'kind' ] = ToBytes( kind[ 0 ].lower() )
   if 'detailed_info' in completion_data:
-    vim_data[ 'info' ] = ToUtf8IfNeeded( completion_data[ 'detailed_info' ] )
+    vim_data[ 'info' ] = ToBytes( completion_data[ 'detailed_info' ] )
     if doc_string:
       vim_data[ 'info' ] += '\n' + doc_string
   elif doc_string:

python/ycm/client/omni_completion_request.py

@@ -15,7 +15,7 @@
 # You should have received a copy of the GNU General Public License
 # along with YouCompleteMe.  If not, see <http://www.gnu.org/licenses/>.
 
-from ycmd.utils import ToUtf8IfNeeded
+from ycmd.utils import ToBytes
 from ycm.client.completion_request import CompletionRequest
 
 
@@ -46,15 +46,15 @@ def ConvertVimDataToCompletionData( vim_data ):
   completion_data = {}
 
   if 'word' in vim_data:
-    completion_data[ 'insertion_text' ] = ToUtf8IfNeeded( vim_data[ 'word' ] )
+    completion_data[ 'insertion_text' ] = ToBytes( vim_data[ 'word' ] )
   if 'abbr' in vim_data:
-    completion_data[ 'menu_text' ] = ToUtf8IfNeeded( vim_data[ 'abbr' ] )
+    completion_data[ 'menu_text' ] = ToBytes( vim_data[ 'abbr' ] )
   if 'menu' in vim_data:
-    completion_data[ 'extra_menu_info' ] = ToUtf8IfNeeded( vim_data[ 'menu' ] )
+    completion_data[ 'extra_menu_info' ] = ToBytes( vim_data[ 'menu' ] )
   if 'kind' in vim_data:
-    completion_data[ 'kind' ] = [ ToUtf8IfNeeded( vim_data[ 'kind' ] ) ]
+    completion_data[ 'kind' ] = [ ToBytes( vim_data[ 'kind' ] ) ]
   if 'info' in vim_data:
-    completion_data[ 'detailed_info' ] = ToUtf8IfNeeded( vim_data[ 'info' ] )
+    completion_data[ 'detailed_info' ] = ToBytes( vim_data[ 'info' ] )
 
   return completion_data
 

python/ycm/vimsupport.py

@@ -21,7 +21,7 @@ import tempfile
 import json
 import re
 from collections import defaultdict
-from ycmd.utils import ToUtf8IfNeeded
+from ycmd.utils import ToBytes, ToUnicode
 from ycmd import user_options_store
 
 BUFFER_COMMAND_MAP = { 'same-buffer'      : 'edit',
@@ -277,7 +277,7 @@ def ConvertDiagnosticsToQfList( diagnostics ):
       'bufnr' : GetBufferNumberForFilename( location[ 'filepath' ] ),
       'lnum'  : line_num,
       'col'   : location[ 'column_num' ],
-      'text'  : ToUtf8IfNeeded( text ),
+      'text'  : ToBytes( text ),
       'type'  : diagnostic[ 'kind' ][ 0 ],
       'valid' : 1
     }
@@ -409,7 +409,7 @@ def NumLinesInBuffer( buffer_object ):
 # or type command to continue" prompt when editing a new C-family file.
 def PostVimMessage( message ):
   vim.command( "redraw | echohl WarningMsg | echom '{0}' | echohl None"
-               .format( EscapeForVim( str( message ) ) ) )
+               .format( EscapeForVim( ToUnicode( message ) ) ) )
 
 
 # Unlike PostVimMesasge, this supports messages with newlines in them because it
@@ -417,7 +417,7 @@ def PostVimMessage( message ):
 # appear in Vim's message log.
 def PostMultiLineNotice( message ):
   vim.command( "echohl WarningMsg | echo '{0}' | echohl None"
-               .format( EscapeForVim( str( message ) ) ) )
+               .format( EscapeForVim( ToUnicode( message ) ) ) )
 
 
 def PresentDialog( message, choices, default_choice_index = 0 ):
@@ -439,8 +439,10 @@ def PresentDialog( message, choices, default_choice_index = 0 ):
     PresentDialog("Is this a nice example?", ["Yes", "No", "May&be"])
       Is this a nice example?
       [Y]es, (N)o, May(b)e:"""
-  to_eval = "confirm('{0}', '{1}', {2})".format( EscapeForVim( message ),
-    EscapeForVim( "\n" .join( choices ) ), default_choice_index + 1 )
+  to_eval = "confirm('{0}', '{1}', {2})".format(
+    EscapeForVim( ToUnicode( message ) ),
+    EscapeForVim( ToUnicode( "\n" .join( choices ) ) ),
+    default_choice_index + 1 )
   return int( vim.eval( to_eval ) ) - 1
 
 
@@ -453,16 +455,17 @@ def Confirm( message ):
 def EchoText( text, log_as_message = True ):
   def EchoLine( text ):
     command = 'echom' if log_as_message else 'echo'
-    vim.command( "{0} '{1}'".format( command, EscapeForVim( text ) ) )
+    vim.command( "{0} '{1}'".format( command,
+                                     EscapeForVim( text ) ) )
 
-  for line in str( text ).split( '\n' ):
+  for line in ToUnicode( text ).split( '\n' ):
     EchoLine( line )
 
 
 # Echos text but truncates it so that it all fits on one line
 def EchoTextVimWidth( text ):
   vim_width = GetIntValue( '&columns' )
-  truncated_text = ToUtf8IfNeeded( text )[ : int( vim_width * 0.9 ) ]
+  truncated_text = ToUnicode( text )[ : int( vim_width * 0.9 ) ]
   truncated_text.replace( '\n', ' ' )
 
   old_ruler = GetIntValue( '&ruler' )

python/ycm/youcompleteme.py

@@ -117,7 +117,7 @@ class YouCompleteMe( object ):
                '--idle_suicide_seconds={0}'.format(
                   SERVER_IDLE_SUICIDE_SECONDS )]
 
-      filename_format = os.path.join( utils.PathToTempDir(),
+      filename_format = os.path.join( utils.PathToCreatedTempDir(),
                                       'server_{port}_{std}.log' )
 
       self._server_stdout = filename_format.format( port = server_port,
@@ -131,12 +131,13 @@ class YouCompleteMe( object ):
         args.append( '--keep_logfiles' )
 
       self._server_popen = utils.SafePopen( args, stdin_windows = PIPE,
-                                            stdout = PIPE, stderr = PIPE)
+                                            stdout = PIPE, stderr = PIPE )
       BaseRequest.server_location = 'http://127.0.0.1:' + str( server_port )
       BaseRequest.hmac_secret = hmac_secret
 
     self._NotifyUserIfServerCrashed()
 
+
   def IsServerAlive( self ):
     returncode = self._server_popen.poll()
     # When the process hasn't finished yet, poll() returns None.

run_tests.py

@@ -13,6 +13,17 @@ python_path = []
 for folder in os.listdir( DIR_OF_THIRD_PARTY ):
   python_path.append( p.abspath( p.join( DIR_OF_THIRD_PARTY, folder ) ) )
 for folder in os.listdir( DIR_OF_YCMD_THIRD_PARTY ):
+  # We skip python-future because it needs to be inserted in sys.path AFTER
+  # the standard library imports but we can't do that with PYTHONPATH because
+  # the std lib paths are always appended to PYTHONPATH. We do it correctly in
+  # prod in ycmd/utils.py because we have access to the right sys.path.
+  # So for dev, we rely on python-future being installed correctly with
+  #   pip install -r test_requirements.txt
+  #
+  # Pip knows how to install this correctly so that it doesn't matter where in
+  # sys.path the path is.
+  if folder == 'python-future':
+    continue
   python_path.append( p.abspath( p.join( DIR_OF_YCMD_THIRD_PARTY, folder ) ) )
 if os.environ.get( 'PYTHONPATH' ):
   python_path.append( os.environ[ 'PYTHONPATH' ] )

third_party/ycmd

@@ -1 +1 @@
-Subproject commit d0932eb1895644e982d3a1daf210c09ca479a620
+Subproject commit 3ed1634afd7bfef872abde4e91bd012fbb08a9d8