Initial support for inefficient semantic highlighting

Receive tokens asynchronously
Allow overriding the property types
Don't apply the highlights if buffer changed... this can lead to errors
Disable by default; add a swtich to enable

autoload/youcompleteme.vim

@@ -40,28 +40,32 @@ let s:previous_allowed_buffer_number = 0
 let s:pollers = {
       \   'completion': {
       \     'id': -1,
-      \     'wait_milliseconds': 10
+      \     'wait_milliseconds': 10,
       \   },
       \   'signature_help': {
       \     'id': -1,
-      \     'wait_milliseconds': 10
+      \     'wait_milliseconds': 10,
       \   },
       \   'file_parse_response': {
       \     'id': -1,
-      \     'wait_milliseconds': 100
+      \     'wait_milliseconds': 100,
       \   },
       \   'server_ready': {
       \     'id': -1,
-      \     'wait_milliseconds': 100
+      \     'wait_milliseconds': 100,
       \   },
       \   'receive_messages': {
       \     'id': -1,
-      \     'wait_milliseconds': 100
+      \     'wait_milliseconds': 100,
       \   },
       \   'command': {
       \     'id': -1,
-      \     'wait_milliseconds': 100
-      \   }
+      \     'wait_milliseconds': 100,
+      \   },
+      \   'semantic_highlighting': {
+      \     'id': -1,
+      \     'wait_milliseconds': 100,
+      \   },
       \ }
 let s:buftype_blacklist = {
       \   'help': 1,
@@ -265,6 +269,7 @@ sys.path[ 0:0 ] = [ p.join( root_folder, 'python' ),
 try:
   # Import the modules used in this file.
   from ycm import base, vimsupport, youcompleteme
+  from ycm import semantic_highlighting as ycm_semantic_highlighting
 
   if 'ycm_state' in globals():
     # If re-initializing, pretend that we shut down
@@ -289,6 +294,7 @@ try:
     default_options = {}
 
   ycm_state = youcompleteme.YouCompleteMe( default_options )
+  ycm_semantic_highlighting.Initialise()
 except Exception as error:
   # We don't use PostVimMessage or EchoText from the vimsupport module because
   # importing this module may fail.
@@ -755,6 +761,15 @@ function! s:OnFileReadyToParse( ... )
     let s:pollers.file_parse_response.id = timer_start(
           \ s:pollers.file_parse_response.wait_milliseconds,
           \ function( 's:PollFileParseResponse' ) )
+
+    if get( g:, 'ycm_enable_semantic_highlightng', 0 ) ||
+          \ get( b:, 'ycm_enable_semantic_highlightng', 0 )
+      py3 ycm_state.CurrentBuffer().SendSemanticTokensRequest()
+      call s:StopPoller( s:pollers.semantic_highlighting )
+      let s:pollers.semantic_highlighting.id = timer_start(
+            \ s:pollers.semantic_highlighting.wait_milliseconds,
+            \ function( 's:PollSemanticHighlighting' ) )
+    endif
   endif
 endfunction
 
@@ -774,6 +789,21 @@ function! s:PollFileParseResponse( ... )
 endfunction
 
 
+function! s:PollSemanticHighlighting( ... )
+  if !py3eval( 'ycm_state.CurrentBuffer().SemanticTokensRequestReady()' )
+    let s:pollers.semantic_highlighting.id = timer_start(
+          \ s:pollers.semantic_highlighting.wait_milliseconds,
+          \ function( 's:PollSemanticHighlighting' ) )
+  endif
+
+  if py3eval( 'ycm_state.CurrentBuffer().UpdateSemanticTokens()' )
+    let s:pollers.semantic_highlighting.id = timer_start(
+          \ s:pollers.semantic_highlighting.wait_milliseconds,
+          \ function( 's:PollSemanticHighlighting' ) )
+  endif
+endfunction
+
+
 function! s:SendKeys( keys )
   " By default keys are added to the end of the typeahead buffer. If there are
   " already keys in the buffer, they will be processed first and may change

python/ycm/buffer.py

@@ -18,6 +18,7 @@
 from ycm import vimsupport
 from ycm.client.event_notification import EventNotification
 from ycm.diagnostic_interface import DiagnosticInterface
+from ycm.semantic_highlighting import SemanticHighlighting
 
 
 # Emulates Vim buffer
@@ -35,6 +36,8 @@ class Buffer:
     self._diag_interface = DiagnosticInterface( bufnr, user_options )
     self._open_loclist_on_ycm_diags = user_options[
                                         'open_loclist_on_ycm_diags' ]
+    self._semantic_highlighting = SemanticHighlighting( bufnr,
+                                                        user_options )
     self.UpdateFromFileTypes( filetypes )
 
 
@@ -133,6 +136,18 @@ class Buffer:
     self._async_diags = False
 
 
+  def SendSemanticTokensRequest( self ):
+    self._semantic_highlighting.SendRequest()
+
+
+  def SemanticTokensRequestReady( self ):
+    return self._semantic_highlighting.IsResponseReady()
+
+
+  def UpdateSemanticTokens( self ):
+    return self._semantic_highlighting.Update()
+
+
   def _ChangedTick( self ):
     return vimsupport.GetBufferChangedTick( self._number )
 

python/ycm/client/semantic_tokens_request.py

@@ -0,0 +1,64 @@
+# Copyright (C) 2020, YouCompleteMe Contributors
+#
+# This file is part of YouCompleteMe.
+#
+# YouCompleteMe is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# YouCompleteMe is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with YouCompleteMe.  If not, see <http://www.gnu.org/licenses/>.
+
+
+import logging
+from ycm.client.base_request import ( BaseRequest, DisplayServerException,
+                                      MakeServerException )
+
+_logger = logging.getLogger( __name__ )
+
+
+# FIXME: This is copy/pasta from SignatureHelpRequest - abstract a
+# SimpleAsyncRequest base that does all of this generically
+class SemanticTokensRequest( BaseRequest ):
+  def __init__( self, request_data ):
+    super().__init__()
+    self.request_data = request_data
+    self._response_future = None
+
+
+  def Start( self ):
+    self._response_future = self.PostDataToHandlerAsync( self.request_data,
+                                                         'semantic_tokens' )
+
+  def Done( self ):
+    return bool( self._response_future ) and self._response_future.done()
+
+
+  def Reset( self ):
+    self._response_future = None
+
+  def Response( self ):
+    if not self._response_future:
+      return {}
+
+    response = self.HandleFuture( self._response_future,
+                                  truncate_message = True )
+
+    if not response:
+      return {}
+
+    # Vim may not be able to convert the 'errors' entry to its internal format
+    # so we remove it from the response.
+    errors = response.pop( 'errors', [] )
+    for e in errors:
+      exception = MakeServerException( e )
+      _logger.error( exception )
+      DisplayServerException( exception, truncate_message = True )
+
+    return response.get( 'semantic_tokens' ) or {}

python/ycm/semantic_highlighting.py

@@ -0,0 +1,181 @@
+# Copyright (C) 2020, YouCompleteMe Contributors
+#
+# This file is part of YouCompleteMe.
+#
+# YouCompleteMe is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# YouCompleteMe is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with YouCompleteMe.  If not, see <http://www.gnu.org/licenses/>.
+
+
+from ycm.client.semantic_tokens_request import SemanticTokensRequest
+from ycm.client.base_request import BuildRequestData
+from ycm import vimsupport
+from ycmd import utils
+
+import vim
+import json
+
+
+HIGHLIGHT_GROUP = {
+  'namespace': 'Type',
+  'type': 'Type',
+  'class': 'Structure',
+  'enum': 'Structure',
+  'interface': 'Structure',
+  'struct': 'Structure',
+  'typeParameter': 'Identifier',
+  'parameter': 'Identifier',
+  'variable': 'Identifier',
+  'property': 'Identifier',
+  'enumMember': 'Identifier',
+  'enumConstant': 'Constant',
+  'event': 'Identifier',
+  'function': 'Function',
+  'member': 'Identifier',
+  'macro': 'Macro',
+  'keyword': 'Keyword',
+  'modifier': 'Keyword',
+  'comment': 'Comment',
+  'string': 'String',
+  'number': 'Number',
+  'regexp': 'String',
+  'operator': 'Operator',
+}
+
+
+def Initialise():
+  props = GetTextPropertyTypes()
+  if 'YCM_HL_UNKNOWN' not in props:
+    AddTextPropertyType( 'YCM_HL_UNKNOWN', highlight = 'WarningMsg' )
+
+  for token_type, group in HIGHLIGHT_GROUP.items():
+    prop = f'YCM_HL_{ token_type }'
+    if prop not in props:
+      AddTextPropertyType( prop, highlight = group )
+
+
+# "arbitrary" base id
+NEXT_TEXT_PROP_ID = 70784
+
+
+def NextPropID():
+  global NEXT_TEXT_PROP_ID
+  try:
+    return NEXT_TEXT_PROP_ID
+  finally:
+    NEXT_TEXT_PROP_ID += 1
+
+
+
+class SemanticHighlighting:
+  """Stores the semantic highlighting state for a Vim buffer"""
+
+  def __init__( self, bufnr, user_options ):
+    self._request = None
+    self._bufnr = bufnr
+    self._prop_id = NextPropID()
+    self.tick = -1
+
+
+  def SendRequest( self ):
+    if self._request and not self.IsResponseReady():
+      return
+
+    self.tick = vimsupport.GetBufferChangedTick( self._bufnr )
+    self._request = SemanticTokensRequest( BuildRequestData() )
+    self._request.Start()
+
+  def IsResponseReady( self ):
+    return self._request is not None and self._request.Done()
+
+  def Update( self ):
+    if not self.IsResponseReady():
+      # Not ready - poll
+      return False
+
+    if self.tick != vimsupport.GetBufferChangedTick( self._bufnr ):
+      # Buffer has changed, we should ignore the data and retry
+      # self.SendRequest()
+      return False
+
+    # We requested a snapshot
+    response = self._request.Response()
+    self._request = None
+
+    tokens = response.get( 'tokens', [] )
+
+    prev_prop_id = self._prop_id
+    self._prop_id = NextPropID()
+
+    for token in tokens:
+      if token[ 'type' ] not in HIGHLIGHT_GROUP:
+        continue
+      prop_type = f"YCM_HL_{ token[ 'type' ] }"
+      AddTextProperty( self._bufnr, self._prop_id, prop_type, token[ 'range' ] )
+
+    ClearTextProperties( self._bufnr, prev_prop_id )
+
+    # No need to re-poll
+    return False
+
+
+# FIXME/TODO: Merge this with vimsupport funcitons, added after these were
+# writted for Diagnostics
+
+if not vimsupport.VimSupportsTextProperties():
+  def AddTextPropertyType( *args, **kwargs ):
+    pass
+  def GetTextPropertyTypes( *args, **kwargs ):
+    return []
+  def AddTextProperty( *args, **kwargs ):
+    pass
+  def ClearTextProperties( *args, **kwargs ):
+    pass
+
+else:
+  def AddTextPropertyType( name, **kwargs ):
+    props = {
+      'highlight': 'Ignore',
+      'combine': False,
+      'start_incl': False,
+      'end_incl': False,
+      'priority': 10
+    }
+    props.update( kwargs )
+
+    vim.eval( f"prop_type_add( '{ vimsupport.EscapeForVim( name ) }', "
+              f"               { json.dumps( kwargs ) } )" )
+
+
+  def GetTextPropertyTypes( *args, **kwargs ):
+    return [ utils.ToUnicode( p ) for p in vim.eval( 'prop_type_list()' ) ]
+
+
+  def AddTextProperty( bufnr, prop_id, prop_type, range ):
+    props = {
+      'end_lnum': range[ 'end' ][ 'line_num' ],
+      'end_col': range[ 'end' ][ 'column_num' ],
+      'bufnr': bufnr,
+      'id': prop_id,
+      'type': prop_type
+    }
+    vim.eval( f"prop_add( { range[ 'start' ][ 'line_num' ] },"
+              f"          { range[ 'start' ][ 'column_num' ] },"
+              f"          { json.dumps( props ) } )" )
+
+  def ClearTextProperties( bufnr, prop_id ):
+    props = {
+      'id': prop_id,
+      'bufnr': bufnr,
+      'all': 1,
+    }
+    vim.eval( f"prop_remove( { json.dumps( props ) } )" )

python/ycm/vimsupport.py

@@ -1352,6 +1352,11 @@ def HasFastPropList():
   return GetBoolValue( 'has( "patch-8.2.3652" )' )
 
 
+@memoize()
+def VimSupportsTextProperties():
+  return VimHasFunctions( 'prop_add', 'prop_type_add' )
+
+
 @memoize()
 def VimSupportsPopupWindows():
   return VimHasFunctions( 'popup_create',
@@ -1360,9 +1365,7 @@ def VimSupportsPopupWindows():
                           'popup_hide',
                           'popup_settext',
                           'popup_show',
-                          'popup_close',
-                          'prop_add',
-                          'prop_type_add' )
+                          'popup_close' ) and VimSupportsTextProperties()
 
 
 @memoize()