Custom identifier support now done.

This commit is the YCM-client part of the support. The ycmd support is
already done.

We now need per-language identifier regexes in ycmd (see
identifier_utils.py). There's some for HTML, CSS and the generic regex
that was used for everything until now.  Pull requests welcome for other
languages.

Fixes #86.

python/ycm/base.py

@@ -17,11 +17,10 @@
 # You should have received a copy of the GNU General Public License
 # along with YouCompleteMe.  If not, see <http://www.gnu.org/licenses/>.
 
-import vim
 from ycm import vimsupport
-from ycmd import utils
 from ycmd import user_options_store
 from ycmd import request_wrap
+from ycmd import identifier_utils
 import ycm_client_support
 
 YCM_VAR_PREFIX = 'ycm_'
@@ -57,7 +56,9 @@ def LoadJsonDefaultsIntoVim():
 
 def CompletionStartColumn():
   return ( request_wrap.CompletionStartColumn(
-      vim.current.line, vimsupport.CurrentColumn() + 1 ) - 1 )
+      vimsupport.CurrentLineContents(),
+      vimsupport.CurrentColumn() + 1,
+      vimsupport.CurrentFiletypes()[ 0 ] ) - 1 )
 
 
 def CurrentIdentifierFinished():
@@ -65,35 +66,27 @@ def CurrentIdentifierFinished():
   previous_char_index = current_column - 1
   if previous_char_index < 0:
     return True
-  line = vim.current.line
-  try:
-    previous_char = line[ previous_char_index ]
-  except IndexError:
-    return False
+  line = vimsupport.CurrentLineContents()
+  filetype = vimsupport.CurrentFiletypes()[ 0 ]
+  regex = identifier_utils.IdentifierRegexForFiletype( filetype )
 
-  if utils.IsIdentifierChar( previous_char ):
-    return False
-
-  if ( not utils.IsIdentifierChar( previous_char ) and
-       previous_char_index > 0 and
-       utils.IsIdentifierChar( line[ previous_char_index - 1 ] ) ):
-    return True
-  else:
-    return line[ : current_column ].isspace()
+  for match in regex.finditer( line ):
+    if match.end() == previous_char_index:
+      return True
+  # If the whole line is whitespace, that means the user probably finished an
+  # identifier on the previous line.
+  return line[ : current_column ].isspace()
 
 
 def LastEnteredCharIsIdentifierChar():
   current_column = vimsupport.CurrentColumn()
-  previous_char_index = current_column - 1
-  if previous_char_index < 0:
+  if current_column - 1 < 0:
     return False
-  line = vim.current.line
-  try:
-    previous_char = line[ previous_char_index ]
-  except IndexError:
-    return False
-
-  return utils.IsIdentifierChar( previous_char )
+  line = vimsupport.CurrentLineContents()
+  filetype = vimsupport.CurrentFiletypes()[ 0 ]
+  return (
+    identifier_utils.StartOfLongestIdentifierEndingAtIndex(
+        line, current_column, filetype ) != current_column )
 
 
 def AdjustCandidateInsertionText( candidates ):
@@ -178,7 +171,7 @@ def OverlapLength( left_string, right_string ):
       length += 1
 
 
-COMPATIBLE_WITH_CORE_VERSION = 11
+COMPATIBLE_WITH_CORE_VERSION = 12
 
 def CompatibleWithYcmCore():
   try:

python/ycm/tests/base_test.py

@@ -18,38 +18,56 @@
 # You should have received a copy of the GNU General Public License
 # along with YouCompleteMe.  If not, see <http://www.gnu.org/licenses/>.
 
-from nose.tools import eq_
+from nose.tools import eq_, ok_, with_setup
 from mock import MagicMock
 from ycm.test_utils import MockVimModule
 vim_mock = MockVimModule()
 from ycm import base
 from ycm import vimsupport
+import sys
 
+# column is 0-based
+def SetVimCurrentColumnAndLineValue( column, line_value ):
+  vimsupport.CurrentColumn = MagicMock( return_value = column )
+  vimsupport.CurrentLineContents = MagicMock( return_value = line_value )
 
+
+def Setup():
+  sys.modules[ 'ycm.vimsupport' ] = MagicMock()
+  vimsupport.CurrentFiletypes = MagicMock( return_value = [''] )
+  vimsupport.CurrentColumn = MagicMock( return_value = 1 )
+  vimsupport.CurrentLineContents = MagicMock( return_value = '' )
+
+
+@with_setup( Setup )
 def AdjustCandidateInsertionText_Basic_test():
   vimsupport.TextAfterCursor = MagicMock( return_value = 'bar' )
   eq_( [ { 'abbr': 'foobar', 'word': 'foo' } ],
        base.AdjustCandidateInsertionText( [ 'foobar' ] ) )
 
 
+@with_setup( Setup )
 def AdjustCandidateInsertionText_ParenInTextAfterCursor_test():
   vimsupport.TextAfterCursor = MagicMock( return_value = 'bar(zoo' )
   eq_( [ { 'abbr': 'foobar', 'word': 'foo' } ],
        base.AdjustCandidateInsertionText( [ 'foobar' ] ) )
 
 
+@with_setup( Setup )
 def AdjustCandidateInsertionText_PlusInTextAfterCursor_test():
   vimsupport.TextAfterCursor = MagicMock( return_value = 'bar+zoo' )
   eq_( [ { 'abbr': 'foobar', 'word': 'foo' } ],
        base.AdjustCandidateInsertionText( [ 'foobar' ] ) )
 
 
+@with_setup( Setup )
 def AdjustCandidateInsertionText_WhitespaceInTextAfterCursor_test():
   vimsupport.TextAfterCursor = MagicMock( return_value = 'bar zoo' )
   eq_( [ { 'abbr': 'foobar', 'word': 'foo' } ],
        base.AdjustCandidateInsertionText( [ 'foobar' ] ) )
 
 
+@with_setup( Setup )
 def AdjustCandidateInsertionText_MoreThanWordMatchingAfterCursor_test():
   vimsupport.TextAfterCursor = MagicMock( return_value = 'bar.h' )
   eq_( [ { 'abbr': 'foobar.h', 'word': 'foo' } ],
@@ -60,12 +78,14 @@ def AdjustCandidateInsertionText_MoreThanWordMatchingAfterCursor_test():
        base.AdjustCandidateInsertionText( [ 'foobar(zoo' ] ) )
 
 
+@with_setup( Setup )
 def AdjustCandidateInsertionText_NotSuffix_test():
   vimsupport.TextAfterCursor = MagicMock( return_value = 'bar' )
   eq_( [ { 'abbr': 'foofoo', 'word': 'foofoo' } ],
        base.AdjustCandidateInsertionText( [ 'foofoo' ] ) )
 
 
+@with_setup( Setup )
 def AdjustCandidateInsertionText_NothingAfterCursor_test():
   vimsupport.TextAfterCursor = MagicMock( return_value = '' )
   eq_( [ 'foofoo',
@@ -74,6 +94,7 @@ def AdjustCandidateInsertionText_NothingAfterCursor_test():
                                             'zobar' ] ) )
 
 
+@with_setup( Setup )
 def AdjustCandidateInsertionText_MultipleStrings_test():
   vimsupport.TextAfterCursor = MagicMock( return_value = 'bar' )
   eq_( [ { 'abbr': 'foobar', 'word': 'foo' },
@@ -87,6 +108,7 @@ def AdjustCandidateInsertionText_MultipleStrings_test():
                                             'bar' ] ) )
 
 
+@with_setup( Setup )
 def AdjustCandidateInsertionText_DictInput_test():
   vimsupport.TextAfterCursor = MagicMock( return_value = 'bar' )
   eq_( [ { 'abbr': 'foobar', 'word': 'foo' } ],
@@ -94,6 +116,7 @@ def AdjustCandidateInsertionText_DictInput_test():
          [ { 'word': 'foobar' } ] ) )
 
 
+@with_setup( Setup )
 def AdjustCandidateInsertionText_DontTouchAbbr_test():
   vimsupport.TextAfterCursor = MagicMock( return_value = 'bar' )
   eq_( [ { 'abbr': '1234', 'word': 'foo' } ],
@@ -101,40 +124,156 @@ def AdjustCandidateInsertionText_DontTouchAbbr_test():
          [ { 'abbr': '1234', 'word': 'foobar' } ] ) )
 
 
+@with_setup( Setup )
 def OverlapLength_Basic_test():
   eq_( 3, base.OverlapLength( 'foo bar', 'bar zoo' ) )
   eq_( 3, base.OverlapLength( 'foobar', 'barzoo' ) )
 
 
+@with_setup( Setup )
 def OverlapLength_BasicWithUnicode_test():
   eq_( 3, base.OverlapLength( u'bar fäö', u'fäö bar' ) )
   eq_( 3, base.OverlapLength( u'zoofäö', u'fäözoo' ) )
 
 
+@with_setup( Setup )
 def OverlapLength_OneCharOverlap_test():
   eq_( 1, base.OverlapLength( 'foo b', 'b zoo' ) )
 
 
+@with_setup( Setup )
 def OverlapLength_SameStrings_test():
   eq_( 6, base.OverlapLength( 'foobar', 'foobar' ) )
 
 
+@with_setup( Setup )
 def OverlapLength_Substring_test():
   eq_( 6, base.OverlapLength( 'foobar', 'foobarzoo' ) )
   eq_( 6, base.OverlapLength( 'zoofoobar', 'foobar' ) )
 
 
+@with_setup( Setup )
 def OverlapLength_LongestOverlap_test():
   eq_( 7, base.OverlapLength( 'bar foo foo', 'foo foo bar' ) )
 
 
+@with_setup( Setup )
 def OverlapLength_EmptyInput_test():
   eq_( 0, base.OverlapLength( '', 'goobar' ) )
   eq_( 0, base.OverlapLength( 'foobar', '' ) )
   eq_( 0, base.OverlapLength( '', '' ) )
 
 
+@with_setup( Setup )
 def OverlapLength_NoOverlap_test():
   eq_( 0, base.OverlapLength( 'foobar', 'goobar' ) )
   eq_( 0, base.OverlapLength( 'foobar', '(^($@#$#@' ) )
   eq_( 0, base.OverlapLength( 'foo bar zoo', 'foo zoo bar' ) )
+
+
+@with_setup( Setup )
+def LastEnteredCharIsIdentifierChar_Basic_test():
+  SetVimCurrentColumnAndLineValue( 3, 'abc' )
+  ok_( base.LastEnteredCharIsIdentifierChar() )
+
+  SetVimCurrentColumnAndLineValue( 2, 'abc' )
+  ok_( base.LastEnteredCharIsIdentifierChar() )
+
+  SetVimCurrentColumnAndLineValue( 1, 'abc' )
+  ok_( base.LastEnteredCharIsIdentifierChar() )
+
+
+@with_setup( Setup )
+def LastEnteredCharIsIdentifierChar_FiletypeHtml_test():
+  SetVimCurrentColumnAndLineValue( 3, 'ab-' )
+  vimsupport.CurrentFiletypes = MagicMock( return_value = ['html'] )
+  ok_( base.LastEnteredCharIsIdentifierChar() )
+
+
+@with_setup( Setup )
+def LastEnteredCharIsIdentifierChar_ColumnIsZero_test():
+  SetVimCurrentColumnAndLineValue( 0, 'abc' )
+  ok_( not base.LastEnteredCharIsIdentifierChar() )
+
+
+@with_setup( Setup )
+def LastEnteredCharIsIdentifierChar_LineEmpty_test():
+  SetVimCurrentColumnAndLineValue( 3, '' )
+  ok_( not base.LastEnteredCharIsIdentifierChar() )
+
+  SetVimCurrentColumnAndLineValue( 0, '' )
+  ok_( not base.LastEnteredCharIsIdentifierChar() )
+
+
+@with_setup( Setup )
+def LastEnteredCharIsIdentifierChar_NotIdentChar_test():
+  SetVimCurrentColumnAndLineValue( 3, 'ab;' )
+  ok_( not base.LastEnteredCharIsIdentifierChar() )
+
+  SetVimCurrentColumnAndLineValue( 1, ';' )
+  ok_( not base.LastEnteredCharIsIdentifierChar() )
+
+  SetVimCurrentColumnAndLineValue( 3, 'ab-' )
+  ok_( not base.LastEnteredCharIsIdentifierChar() )
+
+
+@with_setup( Setup )
+def CurrentIdentifierFinished_Basic_test():
+  SetVimCurrentColumnAndLineValue( 3, 'ab;' )
+  ok_( base.CurrentIdentifierFinished() )
+
+  SetVimCurrentColumnAndLineValue( 2, 'ab;' )
+  ok_( not base.CurrentIdentifierFinished() )
+
+  SetVimCurrentColumnAndLineValue( 1, 'ab;' )
+  ok_( not base.CurrentIdentifierFinished() )
+
+
+@with_setup( Setup )
+def CurrentIdentifierFinished_NothingBeforeColumn_test():
+  SetVimCurrentColumnAndLineValue( 0, 'ab;' )
+  ok_( base.CurrentIdentifierFinished() )
+
+  SetVimCurrentColumnAndLineValue( 0, '' )
+  ok_( base.CurrentIdentifierFinished() )
+
+
+@with_setup( Setup )
+def CurrentIdentifierFinished_InvalidColumn_test():
+  SetVimCurrentColumnAndLineValue( 5, '' )
+  ok_( not base.CurrentIdentifierFinished() )
+
+  SetVimCurrentColumnAndLineValue( 5, 'abc' )
+  ok_( not base.CurrentIdentifierFinished() )
+
+
+@with_setup( Setup )
+def CurrentIdentifierFinished_InMiddleOfLine_test():
+  SetVimCurrentColumnAndLineValue( 4, 'bar.zoo' )
+  ok_( base.CurrentIdentifierFinished() )
+
+  SetVimCurrentColumnAndLineValue( 4, 'bar(zoo' )
+  ok_( base.CurrentIdentifierFinished() )
+
+  SetVimCurrentColumnAndLineValue( 4, 'bar-zoo' )
+  ok_( base.CurrentIdentifierFinished() )
+
+
+@with_setup( Setup )
+def CurrentIdentifierFinished_Html_test():
+  SetVimCurrentColumnAndLineValue( 4, 'bar-zoo' )
+  vimsupport.CurrentFiletypes = MagicMock( return_value = ['html'] )
+  ok_( not base.CurrentIdentifierFinished() )
+
+
+@with_setup( Setup )
+def CurrentIdentifierFinished_WhitespaceOnly_test():
+  SetVimCurrentColumnAndLineValue( 1, '\n' )
+  ok_( base.CurrentIdentifierFinished() )
+
+  SetVimCurrentColumnAndLineValue( 3, '\n    ' )
+  ok_( base.CurrentIdentifierFinished() )
+
+  SetVimCurrentColumnAndLineValue( 3, '\t\t\t\t' )
+  ok_( base.CurrentIdentifierFinished() )
+

python/ycm/vimsupport.py

@@ -51,6 +51,10 @@ def CurrentColumn():
   return vim.current.window.cursor[ 1 ]
 
 
+def CurrentLineContents():
+  return vim.current.line
+
+
 def TextAfterCursor():
   """Returns the text after CurrentColumn."""
   return vim.current.line[ CurrentColumn(): ]

third_party/ycmd

@@ -1 +1 @@
-Subproject commit a15659e731d8e8447590d8d1991ab7ba2fae357c
+Subproject commit d1fbef7b8664f75235c073b8743e8b0114f13d69