Can now seed identifier database with keywords

By toggling the g:ycm_seed_identifiers_with_syntax option, the user can now tell
YCM to seed the identifier database with the language's keywords.

This is off by default because it can be noisy. Since the identifier completer
collects identifiers from buffers as the user visits them, the keywords that the
user cares about will already be in the database, regardless of the state of the
new option. So the only keywords added will be the ones the user is not using.

Meh. But people want it so there.

Fixes #142.

README.md

@@ -749,6 +749,20 @@ Default: `1`
 
     let g:ycm_collect_identifiers_from_tags_files = 1
 
+### The `g:ycm_seed_identifiers_with_syntax` option
+
+When this option is set to `1`, YCM's identifier completer will seed its
+identifier database with the keywords of the programming language you're
+writing.
+
+Since the keywords are extracted from the Vim syntax file for the filetype, all
+keywords may not be collected, depending on how the syntax file was written.
+Usually at least 95% of the keywords are successfully extracted.
+
+Default: `0`
+
+    let g:ycm_seed_identifiers_with_syntax = 0
+
 ### The `g:ycm_add_preview_to_completeopt` option
 
 When this option is set to `1`, YCM will add the `preview` string to Vim's

plugin/youcompleteme.vim

@@ -100,6 +100,9 @@ let g:ycm_collect_identifiers_from_comments_and_strings =
 let g:ycm_collect_identifiers_from_tags_files =
       \ get( g:, 'ycm_collect_identifiers_from_tags_files', 1 )
 
+let g:ycm_seed_identifiers_with_syntax =
+      \ get( g:, 'ycm_seed_identifiers_with_syntax', 0 )
+
 let g:ycm_autoclose_preview_window_after_completion =
       \ get( g:, 'ycm_autoclose_preview_window_after_completion', 0 )
 

python/ycm/completers/all/identifier_completer.py

@@ -22,12 +22,14 @@ import vim
 import ycm_core
 from collections import defaultdict
 from ycm.completers.general_completer import GeneralCompleter
+from ycm.completers.general import syntax_parse
 from ycm import vimsupport
 from ycm import utils
 
 MAX_IDENTIFIER_COMPLETIONS_RETURNED = 10
 MIN_NUM_CHARS = int( vimsupport.GetVariableValue(
   "g:ycm_min_num_of_chars_for_completion" ) )
+SYNTAX_FILENAME = 'YCM_PLACEHOLDER_FOR_SYNTAX'
 
 
 class IdentifierCompleter( GeneralCompleter ):
@@ -36,6 +38,7 @@ class IdentifierCompleter( GeneralCompleter ):
     self.completer = ycm_core.IdentifierCompleter()
     self.completer.EnableThreading()
     self.tags_file_last_mtime = defaultdict( int )
+    self.filetypes_with_keywords_loaded = set()
 
 
   def ShouldUseNow( self, start_column ):
@@ -82,6 +85,7 @@ class IdentifierCompleter( GeneralCompleter ):
 
 
   def AddBufferIdentifiers( self ):
+    # TODO: use vimsupport.GetFiletypes; also elsewhere in file
     filetype = vim.eval( "&filetype" )
     filepath = vim.eval( "expand('%:p')" )
     collect_from_comments_and_strings = vimsupport.GetBoolValue(
@@ -126,12 +130,33 @@ class IdentifierCompleter( GeneralCompleter ):
       absolute_paths_to_tag_files )
 
 
+  def AddIdentifiersFromSyntax( self ):
+    filetype = vim.eval( "&filetype" )
+    if filetype in self.filetypes_with_keywords_loaded:
+      return
+
+    self.filetypes_with_keywords_loaded.add( filetype )
+
+    keyword_set = syntax_parse.SyntaxKeywordsForCurrentBuffer()
+    keywords = ycm_core.StringVec()
+    for keyword in keyword_set:
+      keywords.append( keyword )
+
+    filepath = SYNTAX_FILENAME + filetype
+    self.completer.AddIdentifiersToDatabase( keywords,
+                                             filetype,
+                                             filepath )
+
+
   def OnFileReadyToParse( self ):
     self.AddBufferIdentifiers()
 
     if vimsupport.GetBoolValue( 'g:ycm_collect_identifiers_from_tags_files' ):
       self.AddIdentifiersFromTagFiles()
 
+    if vimsupport.GetBoolValue( 'g:ycm_seed_identifiers_with_syntax' ):
+      self.AddIdentifiersFromSyntax()
+
 
   def OnInsertLeave( self ):
     self.AddIdentifierUnderCursor()

python/ycm/completers/general/syntax_parse.py

@@ -0,0 +1,208 @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2013  Strahinja Val Markovic  <val@markovic.io>
+#
+# This file is part of YouCompleteMe.
+#
+# YouCompleteMe is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# YouCompleteMe is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with YouCompleteMe.  If not, see <http://www.gnu.org/licenses/>.
+
+import re
+import vim
+from ycm import vimsupport
+
+SYNTAX_GROUP_REGEX = re.compile(
+  r"""^
+      (?P<group_name>\w+)
+      \s+
+      xxx
+      \s+
+      (?P<content>.+?)
+      $""",
+  re.VERBOSE )
+
+KEYWORD_REGEX = re.compile( r'^[\w,]+$' )
+
+SYNTAX_ARGUMENT_REGEX = re.compile(
+  r"^\w+=.*$" )
+
+SYNTAX_ARGUMENTS = set([
+  'cchar',
+  'conceal',
+  'contained',
+  'containedin',
+  'nextgroup',
+  'skipempty',
+  'skipnl',
+  'skipwhite',
+  'transparent',
+  'concealends',
+  'contains',
+  'display',
+  'extend',
+  'fold',
+  'oneline',
+  'keepend',
+  'excludenl',
+])
+
+# These are the parent groups from which we want to extract keywords
+ROOT_GROUPS = set([
+  'Statement',
+  'Boolean',
+  'Include',
+  'Type',
+])
+
+
+class SyntaxGroup( object ):
+  def __init__( self, name, lines = None ):
+    self.name     = name
+    self.lines    = lines if lines else []
+    self.children = []
+
+
+def SyntaxKeywordsForCurrentBuffer():
+  vim.command( 'redir => b:ycm_syntax' )
+  vim.command( 'silent! syntax list' )
+  vim.command( 'redir END' )
+  syntax_output = vimsupport.GetVariableValue( 'b:ycm_syntax' )
+  return _KeywordsFromSyntaxListOutput( syntax_output )
+
+
+def _KeywordsFromSyntaxListOutput( syntax_output ):
+  group_name_to_group = _SyntaxGroupsFromOutput( syntax_output )
+  _ConnectGroupChildren( group_name_to_group )
+
+  groups_with_keywords = []
+  for root_group in ROOT_GROUPS:
+    groups_with_keywords.extend(
+      _GetAllDescendentats( group_name_to_group[ root_group ] ) )
+
+  keywords = []
+  for group in groups_with_keywords:
+    keywords.extend( _ExtractKeywordsFromGroup( group ) )
+  return set( keywords )
+
+
+def _SyntaxGroupsFromOutput( syntax_output ):
+  group_name_to_group = _CreateInitialGroupMap()
+  lines               = syntax_output.split( '\n' )
+  looking_for_group   = True
+
+  current_group = None
+  for line in lines:
+    if not line:
+      continue
+
+    match = SYNTAX_GROUP_REGEX.search( line )
+    if match:
+      if looking_for_group:
+        looking_for_group = False
+      else:
+        group_name_to_group[ current_group.name ] = current_group
+
+      current_group = SyntaxGroup( match.group( 'group_name' ),
+                                   [ match.group( 'content').strip() ] )
+    else:
+      if looking_for_group:
+        continue
+
+      if line[ 0 ] == ' ' or line[ 0 ] == '\t':
+        current_group.lines.append( line.strip() )
+
+  if current_group:
+    group_name_to_group[ current_group.name ] = current_group
+  return group_name_to_group
+
+
+def _CreateInitialGroupMap():
+  def AddToGroupMap( name, parent ):
+    new_group = SyntaxGroup( name )
+    group_name_to_group[ name ] = new_group
+    parent.children.append( new_group )
+
+  statement_group = SyntaxGroup( 'Statement' )
+  type_group      = SyntaxGroup( 'Type' )
+
+  # See `:h group-name` for details on how the initial group hierarchy is built
+  group_name_to_group = {
+    'Statement': statement_group,
+    'Type': type_group,
+    'Boolean': SyntaxGroup( 'Boolean' ),
+    'Include': SyntaxGroup( 'Include' )
+  }
+
+  AddToGroupMap( 'Conditional', statement_group )
+  AddToGroupMap( 'Repeat'     , statement_group )
+  AddToGroupMap( 'Label'      , statement_group )
+  AddToGroupMap( 'Operator'   , statement_group )
+  AddToGroupMap( 'Keyword'    , statement_group )
+  AddToGroupMap( 'Exception'  , statement_group )
+
+  AddToGroupMap( 'StorageClass', type_group )
+  AddToGroupMap( 'Structure'   , type_group )
+  AddToGroupMap( 'Typedef'     , type_group )
+
+  return group_name_to_group
+
+
+def _ConnectGroupChildren( group_name_to_group ):
+  def GetParentNames( group ):
+    links_to     = 'links to '
+    parent_names = []
+    for line in group.lines:
+      if line.startswith( links_to ):
+        parent_names.append( line[ len( links_to ): ] )
+    return parent_names
+
+  for group in group_name_to_group.itervalues():
+    parent_names = GetParentNames( group )
+
+    for parent_name in parent_names:
+      try:
+        parent_group = group_name_to_group[ parent_name ]
+      except KeyError:
+        continue
+      parent_group.children.append( group )
+
+
+def _GetAllDescendentats( root_group ):
+  descendants = []
+  for child in root_group.children:
+    descendants.append( child )
+    descendants.extend( _GetAllDescendentats( child ) )
+  return descendants
+
+
+def _ExtractKeywordsFromGroup( group ):
+  keywords = []
+  for line in group.lines:
+    if line.startswith( 'links to ' ):
+      continue
+
+    words = line.split()
+    if not words or words[ 0 ] in SYNTAX_ARGUMENTS:
+      continue
+
+    for word in words:
+      if ( word not in SYNTAX_ARGUMENTS and
+           not SYNTAX_ARGUMENT_REGEX.match( word ) and
+           KEYWORD_REGEX.match( word ) ):
+
+        if word.endswith( ',' ):
+          word = word[ :-1 ]
+        keywords.append( word )
+  return keywords
+
+

python/ycm/completers/general/tests/syntax_parse_test.py

@@ -0,0 +1,262 @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2013  Strahinja Val Markovic  <val@markovic.io>
+#
+# This file is part of YouCompleteMe.
+#
+# YouCompleteMe is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# YouCompleteMe is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with YouCompleteMe.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+from nose.tools import eq_
+from ycm.test_utils import MockVimModule
+vim_mock = MockVimModule()
+from ycm.completers.general import syntax_parse
+
+
+def ContentsOfTestFile( test_file ):
+  dir_of_script = os.path.dirname( os.path.abspath( __file__ ) )
+  full_path_to_test_file = os.path.join( dir_of_script, 'testdata', test_file )
+  return open( full_path_to_test_file ).read()
+
+
+
+def KeywordsFromSyntaxListOutput_PythonSyntax_test():
+  eq_( set(['and', 'IndexError', 'elif', 'BytesWarning', 'ZeroDivisionError',
+            'ImportError', 'is', 'global', 'UnicodeTranslateError',
+            'GeneratorExit', 'BufferError', 'StopIteration', 'as',
+            'SystemError', 'UnicodeError', 'EnvironmentError', 'in', 'EOFError',
+            'LookupError', 'Exception', 'PendingDeprecationWarning', 'if',
+            'OSError', 'DeprecationWarning', 'raise', 'for',
+            'FloatingPointError', 'UnicodeWarning', 'VMSError', 'except',
+            'nonlocal', 'ReferenceError', 'NameError', 'pass', 'finally',
+            'Warning', 'UnboundLocalError', 'print', 'IOError',
+            'IndentationError', 'True', 'RuntimeError', 'FutureWarning',
+            'ImportWarning', 'SystemExit', 'None', 'return', 'StandardError',
+            'exec', 'ValueError', 'TabError', 'else', 'break', 'SyntaxError',
+            'UnicodeEncodeError', 'WindowsError', 'not', 'UnicodeDecodeError',
+            'with', 'class', 'KeyError', 'AssertionError', 'assert',
+            'TypeError', 'False', 'RuntimeWarning', 'KeyboardInterrupt',
+            'UserWarning', 'SyntaxWarning', 'yield', 'OverflowError', 'try',
+            'ArithmeticError', 'while', 'continue', 'del', 'MemoryError',
+            'NotImplementedError', 'BaseException', 'AttributeError', 'or',
+            'def', 'lambda', 'from', 'import']),
+       syntax_parse._KeywordsFromSyntaxListOutput(
+         ContentsOfTestFile( 'python_syntax' ) ) )
+
+
+def KeywordsFromSyntaxListOutput_CppSyntax_test():
+  eq_( set(['int_fast32_t', 'FILE', 'size_t', 'bitor', 'typedef', 'const',
+            'struct', 'uint8_t', 'fpos_t', 'thread_local', 'unsigned',
+            'uint_least16_t', 'match', 'do', 'intptr_t', 'uint_least64_t',
+            'return', 'auto', 'void', '_Complex', 'break', '_Alignof', 'not',
+            'using', '_Static_assert', '_Thread_local', 'public',
+            'uint_fast16_t', 'this', 'continue', 'char32_t', 'int16_t',
+            'intmax_t', 'static', 'clock_t', 'sizeof', 'int_fast64_t',
+            'mbstate_t', 'try', 'xor', 'uint_fast32_t', 'int_least8_t', 'div_t',
+            'volatile', 'template', 'char16_t', 'new', 'ldiv_t',
+            'int_least16_t', 'va_list', 'uint_least8_t', 'goto', 'noreturn',
+            'enum', 'static_assert', 'bitand', 'compl', 'imaginary', 'jmp_buf',
+            'throw', 'asm', 'ptrdiff_t', 'uint16_t', 'or', 'uint_fast8_t',
+            '_Bool', 'int32_t', 'float', 'private', 'restrict', 'wint_t',
+            'operator', 'not_eq', '_Imaginary', 'alignas', 'union', 'long',
+            'uint_least32_t', 'int_least64_t', 'friend', 'uintptr_t', 'int8_t',
+            'else', 'export', 'int_fast8_t', 'catch', 'true', 'case', 'default',
+            'double', '_Noreturn', 'signed', 'typename', 'while', 'protected',
+            'wchar_t', 'wctrans_t', 'uint64_t', 'delete', 'and', 'register',
+            'false', 'int', 'uintmax_t', 'off_t', 'char', 'int64_t',
+            'int_fast16_t', 'DIR', '_Atomic', 'time_t', 'xor_eq', 'namespace',
+            'virtual', 'complex', 'bool', 'mutable', 'if', 'int_least32_t',
+            'sig_atomic_t', 'and_eq', 'ssize_t', 'alignof', '_Alignas',
+            '_Generic', 'extern', 'class', 'typeid', 'short', 'for',
+            'uint_fast64_t', 'wctype_t', 'explicit', 'or_eq', 'switch',
+            'uint32_t', 'inline']),
+       syntax_parse._KeywordsFromSyntaxListOutput(
+         ContentsOfTestFile( 'cpp_syntax' ) ) )
+
+
+def KeywordsFromSyntaxListOutput_JavaSyntax_test():
+  eq_( set(['false', 'synchronized', 'int', 'abstract', 'float', 'private',
+            'char', 'catch', 'boolean', 'static', 'native', 'for', 'super',
+            'while', 'long', 'throw', 'strictfp', 'finally', 'continue',
+            'extends', 'volatile', 'if', 'public', 'match', 'do', 'return',
+            'void', 'enum', 'else', 'break', 'transient', 'new', 'interface',
+            'instanceof', 'byte', 'true', 'serializable', 'implements',
+            'assert', 'short', 'package', 'this', 'double', 'final', 'try',
+            'default', 'switch', 'protected', 'throws']),
+       syntax_parse._KeywordsFromSyntaxListOutput(
+         ContentsOfTestFile( 'java_syntax' ) ) )
+
+
+def KeywordsFromSyntaxListOutput_Basic_test():
+  eq_( set([ 'foo', 'bar', 'zoo', 'goo' ]),
+       syntax_parse._KeywordsFromSyntaxListOutput( """
+foogroup xxx foo bar
+             zoo goo
+             links to Statement"""
+         )
+     )
+
+
+def KeywordsFromSyntaxListOutput_JunkIgnored_test():
+  eq_( set([ 'foo', 'bar', 'zoo', 'goo' ]),
+       syntax_parse._KeywordsFromSyntaxListOutput( """
+--- Syntax items ---
+foogroup xxx foo bar
+             zoo goo
+             links to Statement
+Spell          cluster=NONE
+NoSpell        cluster=NONE"""
+         )
+     )
+
+
+def KeywordsFromSyntaxListOutput_MultipleStatementGroups_test():
+  eq_( set([ 'foo', 'bar', 'zoo', 'goo' ]),
+       syntax_parse._KeywordsFromSyntaxListOutput( """
+foogroup xxx foo bar
+             links to Statement
+bargroup xxx zoo goo
+             links to Statement"""
+         )
+     )
+
+
+def KeywordsFromSyntaxListOutput_StatementAndTypeGroups_test():
+  eq_( set([ 'foo', 'bar', 'zoo', 'goo' ]),
+       syntax_parse._KeywordsFromSyntaxListOutput( """
+foogroup xxx foo bar
+             links to Statement
+bargroup xxx zoo goo
+             links to Type"""
+         )
+     )
+
+
+def KeywordsFromSyntaxListOutput_StatementHierarchy_test():
+  eq_( set([ 'foo', 'bar', 'zoo', 'goo', 'qux', 'moo' ]),
+       syntax_parse._KeywordsFromSyntaxListOutput( """
+baa xxx foo bar
+        links to Foo
+Foo xxx zoo goo
+        links to Bar
+Bar xxx qux moo
+        links to Statement"""
+         )
+     )
+
+
+def KeywordsFromSyntaxListOutput_TypeHierarchy_test():
+  eq_( set([ 'foo', 'bar', 'zoo', 'goo', 'qux', 'moo' ]),
+       syntax_parse._KeywordsFromSyntaxListOutput( """
+baa xxx foo bar
+        links to Foo
+Foo xxx zoo goo
+        links to Bar
+Bar xxx qux moo
+        links to Type"""
+         )
+     )
+
+
+def KeywordsFromSyntaxListOutput_StatementAndTypeHierarchy_test():
+  eq_( set([ 'foo', 'bar', 'zoo', 'goo', 'qux', 'moo', 'na', 'nb', 'nc' ]),
+       syntax_parse._KeywordsFromSyntaxListOutput( """
+tBaa xxx foo bar
+        links to tFoo
+tFoo xxx zoo goo
+        links to tBar
+tBar xxx qux moo
+        links to Type
+sBaa xxx na bar
+        links to sFoo
+sFoo xxx zoo nb
+        links to sBar
+sBar xxx qux nc
+        links to Statement"""
+         )
+     )
+
+
+def SyntaxGroupsFromOutput_Basic_test():
+  groups = syntax_parse._SyntaxGroupsFromOutput(
+        """foogroup xxx foo bar
+                        zoo goo
+                        links to Statement""" )
+
+  assert 'foogroup' in groups
+
+
+def ExtractKeywordsFromGroup_Basic_test():
+  eq_( ['foo', 'bar', 'zoo', 'goo' ],
+       syntax_parse._ExtractKeywordsFromGroup( syntax_parse.SyntaxGroup('', [
+         'foo bar',
+         'zoo goo',
+       ] ) )
+     )
+
+
+def ExtractKeywordsFromGroup_Commas_test():
+  eq_( ['foo', 'bar', 'zoo', 'goo' ],
+       syntax_parse._ExtractKeywordsFromGroup( syntax_parse.SyntaxGroup('', [
+         'foo, bar,',
+         'zoo goo',
+       ] ) )
+     )
+
+
+def ExtractKeywordsFromGroup_WithLinksTo_test():
+  eq_( ['foo', 'bar', 'zoo', 'goo' ],
+       syntax_parse._ExtractKeywordsFromGroup( syntax_parse.SyntaxGroup('', [
+         'foo bar',
+         'zoo goo',
+         'links to Statement'
+       ] ) )
+     )
+
+
+def ExtractKeywordsFromGroup_KeywordStarts_test():
+  eq_( ['foo', 'bar', 'zoo', 'goo' ],
+       syntax_parse._ExtractKeywordsFromGroup( syntax_parse.SyntaxGroup('', [
+         'foo bar',
+         'transparent boo baa',
+         'zoo goo',
+       ] ) )
+     )
+
+
+def ExtractKeywordsFromGroup_KeywordMiddle_test():
+  eq_( ['foo', 'bar', 'zoo', 'goo' ],
+       syntax_parse._ExtractKeywordsFromGroup( syntax_parse.SyntaxGroup('', [
+         'foo oneline bar',
+         'zoo goo',
+       ] ) )
+     )
+
+
+def ExtractKeywordsFromGroup_KeywordAssign_test():
+  eq_( ['foo', 'bar', 'zoo', 'goo' ],
+       syntax_parse._ExtractKeywordsFromGroup( syntax_parse.SyntaxGroup('', [
+         'foo end=zoo((^^//)) bar',
+         'zoo goo',
+       ] ) )
+     )
+
+
+def ExtractKeywordsFromGroup_KeywordAssignAndMiddle_test():
+  eq_( ['foo', 'bar', 'zoo', 'goo' ],
+       syntax_parse._ExtractKeywordsFromGroup( syntax_parse.SyntaxGroup('', [
+         'foo end=zoo((^^//)) transparent bar',
+         'zoo goo',
+       ] ) )
+     )

python/ycm/completers/general/tests/testdata/java_syntax

@@ -0,0 +1,231 @@
+--- Syntax items ---
+htmlValue      xxx match /=[\t ]*[^'" \t>][^ \t>]*/hs=s+1  contained contains=javaScriptExpression,@htmlPreproc
+                   links to Normal
+cCustomFunc    xxx match /\w\+\s*\((\)\@=/
+                   links to Function
+cCustomClass   xxx match /\w\+\s*\(::\)\@=/
+                   links to Function
+OperatorChars  xxx match #?\|+\|-\|\*\|;\|:\|,\|<\|>\|&\||\|!\|\~\|%\|=\|)\|(\|{\|}\|\.\|\[\|\]\|/\(/\|*\)\@!#
+javaFold       xxx start=/{/ end=/}/  transparent fold
+javaError      xxx const goto
+                   match /[\\@`]/
+                   match +<<<\|\.\.\|=>\|||=\|&&=\|[^-]->\|\*\/+
+                   links to Error
+javaOK         xxx match /\.\.\./
+javaError2     xxx match /#\|=</
+                   links to javaError
+javaExternal   xxx native package
+                   match /\<import\>\(\s\+static\>\)\?/
+                   links to Include
+javaConditional xxx if else switch
+                   links to Conditional
+javaRepeat     xxx do for while
+                   links to Repeat
+javaBoolean    xxx true false
+                   links to Boolean
+javaConstant   xxx null
+                   links to Constant
+javaTypedef    xxx this super
+                   match /\.\s*\<class\>/ms=s+1
+                   links to Typedef
+javaOperator   xxx new instanceof
+                   links to Operator
+javaType       xxx float boolean long void double short char byte int
+                   links to Type
+javaStatement  xxx return
+                   links to Statement
+javaStorageClass xxx transient strictfp serializable synchronized static final volatile
+                   links to StorageClass
+javaExceptions xxx finally catch try throw
+                   links to Exception
+javaAssert     xxx assert
+                   links to Statement
+javaMethodDecl xxx synchronized throws
+                   links to javaStorageClass
+javaClassDecl  xxx interface implements enum extends
+                   match /^class\>/
+                   match /[^.]\s*\<class\>/ms=s+1
+                   match /@interface\>/
+                   links to javaStorageClass
+javaAnnotation xxx match /@\([_$a-zA-Z][_$a-zA-Z0-9]*\.\)*[_$a-zA-Z][_$a-zA-Z0-9]*\>/
+                   links to PreProc
+javaBranch     xxx nextgroup=javaUserLabelRef  skipwhite continue
+                   nextgroup=javaUserLabelRef  skipwhite break
+                   links to Conditional
+javaUserLabelRef xxx match /\k\+/  contained
+                   links to javaUserLabel
+javaVarArg     xxx match /\.\.\./
+                   links to Function
+javaScopeDecl  xxx protected public private abstract
+                   links to javaStorageClass
+javaLabel      xxx default
+                   links to Label
+javaNumber     xxx match /\<\(0[0-7]*\|0[xX]\x\+\|\d\+\)[lL]\=\>/
+                   match /\(\<\d\+\.\d*\|\.\d\+\)\([eE][-+]\=\d\+\)\=[fFdD]\=/
+                   match /\<\d\+[eE][-+]\=\d\+[fFdD]\=\>/
+                   match /\<\d\+\([eE][-+]\=\d\+\)\=[fFdD]\>/
+                   links to Number
+javaCharacter  xxx match /'[^']*'/  contains=javaSpecialChar,javaSpecialCharError
+                   match /'\\''/  contains=javaSpecialChar
+                   match /'[^\\]'/
+                   links to Character
+javaLabelRegion xxx matchgroup=javaLabel start=/\<case\>/ matchgroup=NONE end=/:/  transparent contains=javaNumber,javaCharacter
+javaUserLabel  xxx match /^\s*[_$a-zA-Z][_$a-zA-Z0-9_]*\s*:/he=e-1  contains=javaLabel
+                   links to Label
+javaTodo       xxx contained TODO XXX FIXME
+                   links to Todo
+javaSpecial    xxx match /\\u\d\{4\}/
+                   links to Special
+javaCommentStar xxx match +^\s*\*[^/]+me=e-1  contained
+                   match /^\s*\*$/  contained
+                   links to javaComment
+javaSpecialChar xxx match /\\\([4-9]\d\|[0-3]\d\d\|[\"\\'ntbrf]\|u\x\{4\}\)/  contained
+                   links to SpecialChar
+javaComment    xxx start=+/\*+ end=+\*/+  contains=@javaCommentSpecial,javaTodo,@Spell
+                   match +/\*\*/+
+                   links to Comment
+javaLineComment xxx match +//.*+  contains=@javaCommentSpecial2,javaTodo,@Spell
+                   links to Comment
+javaString     xxx start=/"/ end=/$/ end=/"/  contains=javaSpecialChar,javaSpecialError,@Spell
+                   links to String
+htmlError      xxx match /[<>&]/  contained
+                   links to Error
+htmlSpecialChar xxx match /&#\=[0-9A-Za-z]\{1,8};/  contained
+                   links to Special
+htmlString     xxx start=/"/ end=/"/  contained contains=htmlSpecialChar,javaScriptExpression,@htmlPreproc
+                   start=/'/ end=/'/  contained contains=htmlSpecialChar,javaScriptExpression,@htmlPreproc
+                   links to String
+htmlTagN       xxx match /<\s*[-a-zA-Z0-9]\+/hs=s+1  contained contains=htmlTagName,htmlSpecialTagName,@htmlTagNameCluster
+                   match =</\s*[-a-zA-Z0-9]\+=hs=s+2  contained contains=htmlTagName,htmlSpecialTagName,@htmlTagNameCluster
+htmlTagError   xxx match /[^>]</ms=s+1  contained
+                   links to htmlError
+htmlEndTag     xxx start=+</+ end=/>/  contained contains=htmlTagN,htmlTagError
+                   links to Identifier
+htmlArg        xxx contained below color name gutter span classid alt marginheight target rows bgcolor ismap cellspacing object codetype frame noshade data for bordercolor clip rowspan defer cellpadding shape usemap rules multiple start selected language summary hspace lowsrc type valign hreflang noresize scheme
+                   contained class visibility checked pagex pagey headers scrolling clear charset id id declare codebase tabindex standby version link accept coords alink background vspace wrap profile width compact marginwidth above content border top maxlength prompt dir value charoff height longdesc nowrap
+                   contained accesskey cols cite rel rev style method size src axis vlink valuetype colspan nohref face lang frameborder enctype readonly action left text url char align scope code disabled abbr datetime archive
+                   match /\<\(http-equiv\|href\|title\)=/me=e-1  contained
+                   match /\<z-index\>/  contained
+                   match /\<\(accept-charset\|label\)\>/  contained
+                   links to Type
+htmlTag        xxx start=+<[^/]+ end=/>/  contained fold contains=htmlTagN,htmlString,htmlArg,htmlValue,htmlTagError,htmlEvent,htmlCssDefinition,@htmlPreproc,@htmlArgCluster
+                   links to Function
+htmlTagName    xxx contained marquee span ol thead pre blink tbody a p q s object legend frame acronym noframes blockquote var table input hr button bdo font caption sub del sup col basefont xmp iframe dfn html meta nobr fieldset optgroup option applet link area map li td th tr tt param center address small frameset
+                   contained label ul dir div kbd cite big layer form textarea base tfoot br strike samp select menu colgroup img nolayer spacer ilayer code abbr isindex dd ins dl dt noscript
+                   match /\<\(b\|i\|u\|h[1-6]\|em\|strong\|head\|body\|title\)\>/  contained
+                   links to htmlStatement
+htmlSpecialTagName xxx contained script style
+                   links to Exception
+htmlCommentPart xxx start=/--/ end=/--\s*/  contained contains=@htmlPreproc,@Spell
+                   links to Comment
+htmlCommentError xxx match /[^><!]/  contained
+                   links to htmlError
+htmlComment    xxx start=/<!/ end=/>/  contained contains=htmlCommentPart,htmlCommentError,@Spell
+                   start=/<!DOCTYPE/ end=/>/  contained keepend
+                   links to Comment
+htmlPreStmt    xxx match /<!--#\(config\|echo\|exec\|fsize\|flastmod\|include\|printenv\|set\|if\|elif\|else\|endif\|geoguide\)\>/  contained
+                   links to PreProc
+htmlPreError   xxx match /<!--#\S*/ms=s+4  contained
+                   links to Error
+htmlPreAttr    xxx match /\w\+=[^"]\S\+/  contained contains=htmlPreProcAttrError,htmlPreProcAttrName
+                   start=/\w\+="/ skip=/\\\\\|\\"/ end=/"/  contained keepend contains=htmlPreProcAttrName
+                   links to String
+htmlPreProc    xxx start=/<!--#/ end=/-->/  contained contains=htmlPreStmt,htmlPreError,htmlPreAttr
+                   links to PreProc
+htmlPreProcAttrError xxx match /\w\+=/he=e-1  contained
+                   links to Error
+htmlPreProcAttrName xxx match /\(expr\|errmsg\|sizefmt\|timefmt\|var\|cgi\|cmd\|file\|virtual\|value\)=/he=e-1  contained
+                   links to PreProc
+htmlLink       xxx start=/<a\>\_[^>]*\<href\>/ end=+</a>+me=e-4  contained contains=@Spell,htmlTag,htmlEndTag,htmlSpecialChar,htmlPreProc,htmlComment,htmlLeadingSpace,javaScript,@htmlPreproc
+                   links to Underlined
+htmlBoldUnderline xxx start=/<u\>/ end=+</u>+me=e-4  contained contains=@htmlTop,htmlBoldUnderlineItalic
+htmlBoldItalic xxx start=/<i\>/ end=+</i>+me=e-4  contained contains=@htmlTop,htmlBoldItalicUnderline
+                   start=/<em\>/ end=+</em>+me=e-5  contained contains=@htmlTop,htmlBoldItalicUnderline
+htmlBold       xxx start=/<b\>/ end=+</b>+me=e-4  contained contains=@htmlTop,htmlBoldUnderline,htmlBoldItalic
+                   start=/<strong\>/ end=+</strong>+me=e-9  contained contains=@htmlTop,htmlBoldUnderline,htmlBoldItalic
+htmlBoldUnderlineItalic xxx start=/<i\>/ end=+</i>+me=e-4  contained contains=@htmlTop
+                   start=/<em\>/ end=+</em>+me=e-5  contained contains=@htmlTop
+htmlBoldItalicUnderline xxx start=/<u\>/ end=+</u>+me=e-4  contained contains=@htmlTop,htmlBoldUnderlineItalic
+                   links to htmlBoldUnderlineItalic
+htmlUnderlineBold xxx start=/<b\>/ end=+</b>+me=e-4  contained contains=@htmlTop,htmlUnderlineBoldItalic
+                   start=/<strong\>/ end=+</strong>+me=e-9  contained contains=@htmlTop,htmlUnderlineBoldItalic
+                   links to htmlBoldUnderline
+htmlUnderlineItalic xxx start=/<i\>/ end=+</i>+me=e-4  contained contains=@htmlTop,htmlUnderlineItalicBold
+                   start=/<em\>/ end=+</em>+me=e-5  contained contains=@htmlTop,htmlUnderlineItalicBold
+htmlUnderline  xxx start=/<u\>/ end=+</u>+me=e-4  contained contains=@htmlTop,htmlUnderlineBold,htmlUnderlineItalic
+htmlUnderlineBoldItalic xxx start=/<i\>/ end=+</i>+me=e-4  contained contains=@htmlTop
+                   start=/<em\>/ end=+</em>+me=e-5  contained contains=@htmlTop
+                   links to htmlBoldUnderlineItalic
+htmlUnderlineItalicBold xxx start=/<b\>/ end=+</b>+me=e-4  contained contains=@htmlTop
+                   start=/<strong\>/ end=+</strong>+me=e-9  contained contains=@htmlTop
+                   links to htmlBoldUnderlineItalic
+htmlItalicBold xxx start=/<b\>/ end=+</b>+me=e-4  contained contains=@htmlTop,htmlItalicBoldUnderline
+                   start=/<strong\>/ end=+</strong>+me=e-9  contained contains=@htmlTop,htmlItalicBoldUnderline
+                   links to htmlBoldItalic
+htmlItalicUnderline xxx start=/<u\>/ end=+</u>+me=e-4  contained contains=@htmlTop,htmlItalicUnderlineBold
+                   links to htmlUnderlineItalic
+htmlItalic     xxx start=/<i\>/ end=+</i>+me=e-4  contained contains=@htmlTop,htmlItalicBold,htmlItalicUnderline
+                   start=/<em\>/ end=+</em>+me=e-5  contained contains=@htmlTop
+htmlItalicBoldUnderline xxx start=/<u\>/ end=+</u>+me=e-4  contained contains=@htmlTop
+                   links to htmlBoldUnderlineItalic
+htmlItalicUnderlineBold xxx start=/<b\>/ end=+</b>+me=e-4  contained contains=@htmlTop
+                   start=/<strong\>/ end=+</strong>+me=e-9  contained contains=@htmlTop
+                   links to htmlBoldUnderlineItalic
+htmlLeadingSpace xxx match /^\s\+/  contained
+                   links to None
+htmlH1         xxx start=/<h1\>/ end=+</h1>+me=e-5  contained contains=@htmlTop
+                   links to Title
+htmlH2         xxx start=/<h2\>/ end=+</h2>+me=e-5  contained contains=@htmlTop
+                   links to htmlH1
+htmlH3         xxx start=/<h3\>/ end=+</h3>+me=e-5  contained contains=@htmlTop
+                   links to htmlH2
+htmlH4         xxx start=/<h4\>/ end=+</h4>+me=e-5  contained contains=@htmlTop
+                   links to htmlH3
+htmlH5         xxx start=/<h5\>/ end=+</h5>+me=e-5  contained contains=@htmlTop
+                   links to htmlH4
+htmlH6         xxx start=/<h6\>/ end=+</h6>+me=e-5  contained contains=@htmlTop
+                   links to htmlH5
+htmlTitle      xxx start=/<title\>/ end=+</title>+me=e-8  contained contains=htmlTag,htmlEndTag,htmlSpecialChar,htmlPreProc,htmlComment,javaScript,@htmlPreproc
+                   links to Title
+htmlHead       xxx start=/<head\>/ end=/<h[1-6]\>/me=e-3 end=/<body\>/me=e-5 end=+</head>+me=e-7  contained contains=htmlTag,htmlEndTag,htmlSpecialChar,htmlPreProc,htmlComment,htmlLink,htmlTitle,javaScript,cssStyle,@htmlPreproc
+                   links to PreProc
+javaCommentTitle xxx matchgroup=javaDocComment start=+/\*\*+ matchgroup=javaCommentTitle end=+\*/+me=s-1,he=s-1 end=/[^{]@/me=s-2,he=s-1 end=/\.[ \t\r<&]/me=e-1 end=/\.$/  contained keepend contains=@javaHtml,javaCommentStar,javaTodo,@Spell,javaDocTags,javaDocSeeTag
+                   links to SpecialComment
+javaDocTags    xxx start=/{@\(code\|link\|linkplain\|inherit[Dd]oc\|doc[rR]oot\|value\)/ end=/}/  contained
+                   match /@\(param\|exception\|throws\|since\)\s\+\S\+/  contained contains=javaDocParam
+                   match /@\(version\|author\|return\|deprecated\|serial\|serialField\|serialData\)\>/  contained
+                   links to Special
+javaDocSeeTag  xxx matchgroup=javaDocTags start=/@see\s\+/ matchgroup=NONE end=/\_./re=e-1  contained contains=javaDocSeeTagParam
+javaDocComment xxx start=+/\*\*+ end=+\*/+  keepend contains=javaCommentTitle,@javaHtml,javaDocTags,javaDocSeeTag,javaTodo,@Spell
+                   links to Comment
+javaDocParam   xxx match /\s\S\+/  contained
+                   links to Function
+javaDocSeeTagParam xxx match @"\_[^"]\+"\|<a\s\+\_.\{-}</a>\|\(\k\|\.\)*\(#\k\+\((\_[^)]\+)\)\=\)\=@  contained extend
+                   links to Function
+javaSpecialError xxx match /\\./  contained
+                   links to Error
+javaSpecialCharError xxx match /[^']/  contained
+                   links to Error
+javaParenT1    xxx matchgroup=javaParen1 start=/(/ end=/)/  contained transparent contains=@javaTop,javaParenT2
+                   matchgroup=javaParen1 start=/\[/ end=/\]/  contained transparent contains=@javaTop,javaParenT2
+javaParenT     xxx matchgroup=javaParen start=/(/ end=/)/  transparent contains=@javaTop,javaParenT1
+                   matchgroup=javaParen start=/\[/ end=/\]/  transparent contains=@javaTop,javaParenT1
+javaParenT2    xxx matchgroup=javaParen2 start=/(/ end=/)/  contained transparent contains=@javaTop,javaParenT
+                   matchgroup=javaParen2 start=/\[/ end=/\]/  contained transparent contains=@javaTop,javaParenT
+javaParenError xxx match /)/
+                   match /\]/
+                   links to javaError
+javaTop        cluster=javaError,javaError,javaError,javaError2,javaExternal,javaConditional,javaRepeat,javaBoolean,javaConstant,javaTypedef,javaOperator,javaType,javaType,javaStatement,javaStorageClass,javaExceptions,javaAssert,javaMethodDecl,javaClassDecl,javaClassDecl,javaClassDecl,javaAnnotation,javaBranch,javaVarArg,javaScopeDecl,javaLangObject,javaLabel,javaNumber,javaCharacter,javaLabelRegion,javaUserLabel,javaSpecial,javaComment,javaLineComment,javaString,javaStringError
+Spell          cluster=NONE
+javaCommentSpecial cluster=NONE
+javaCommentSpecial2 cluster=NONE
+javaHtml       cluster=htmlError,htmlSpecialChar,htmlEndTag,htmlTag,htmlComment,htmlPreProc,htmlLink,htmlBold,htmlUnderline,htmlItalic,htmlH1,htmlH2,htmlH3,htmlH4,htmlH5,htmlH6,htmlTitle,htmlHead
+htmlPreproc    cluster=NONE
+htmlArgCluster cluster=NONE
+htmlTagNameCluster cluster=NONE
+htmlTop        cluster=@Spell,htmlTag,htmlEndTag,htmlSpecialChar,htmlPreProc,htmlComment,htmlLink,javaScript,@htmlPreproc
+htmlJavaScript cluster=@htmlPreproc
+htmlVbScript   cluster=NONE
+htmlCss        cluster=NONE
+javaClasses    cluster=NONE
+

python/ycm/completers/general/tests/testdata/python_syntax

@@ -0,0 +1,63 @@
+--- Syntax items ---
+cCustomFunc    xxx match /\w\+\s*\((\)\@=/
+                   links to Function
+cCustomClass   xxx match /\w\+\s*\(::\)\@=/
+                   links to Function
+pythonStatement xxx return True lambda
+                   nextgroup=pythonFunction  skipwhite def
+                   del
+                   nextgroup=pythonFunction  skipwhite class
+                   global nonlocal as None, False, yield with print continue break pass assert exec
+                   links to Statement
+pythonFunction xxx match /\%(\%(def\s\|class\s\|@\)\s*\)\@<=\h\%(\w\|\.\)*/  contained
+                   links to Function
+pythonConditional xxx if else elif
+                   links to Conditional
+pythonRepeat   xxx for while
+                   links to Repeat
+pythonOperator xxx or is and in not
+                   links to Operator
+pythonException xxx finally raise except try
+                   links to Exception
+pythonInclude  xxx from import
+                   links to Include
+pythonDecorator xxx match /@/  display nextgroup=pythonFunction skipwhite
+                   links to Define
+pythonTodo     xxx contained NOTE XXX TODO NOTES FIXME
+                   links to Todo
+pythonComment  xxx match /#.*$/  contains=pythonTodo,@Spell
+                   links to Comment
+pythonEscape   xxx match /\\[abfnrtv'"\\]/  contained
+                   match /\\\o\{1,3}/  contained
+                   match /\\x\x\{2}/  contained
+                   match /\%(\\u\x\{4}\|\\U\x\{8}\)/  contained
+                   match /\\N{\a\+\%(\s\a\+\)*}/  contained
+                   match /\\$/
+                   links to Special
+pythonString   xxx start=/[uU]\=\z(['"]\)/ skip=/\\\\\|\\\z1/ end=/\z1/  contains=pythonEscape,@Spell
+                   start=/[uU]\=\z('''\|"""\)/ end=/\z1/  keepend contains=pythonEscape,pythonSpaceError,pythonDoctest,@Spell
+                   links to String
+pythonDoctest  xxx start=/^\s*>>>\s/ end=/^\s*$/  contained contains=ALLBUT,pythonDoctest,@Spell
+                   links to Special
+pythonRawString xxx start=/[uU]\=[rR]\z(['"]\)/ skip=/\\\\\|\\\z1/ end=/\z1/  contains=@Spell
+                   start=/[uU]\=[rR]\z('''\|"""\)/ end=/\z1/  keepend contains=pythonSpaceError,pythonDoctest,@Spell
+                   links to String
+pythonNumber   xxx match /\<0[oO]\=\o\+[Ll]\=\>/
+                   match /\<0[xX]\x\+[Ll]\=\>/
+                   match /\<0[bB][01]\+[Ll]\=\>/
+                   match /\<\%([1-9]\d*\|0\)[Ll]\=\>/
+                   match /\<\d\+[jJ]\>/
+                   match /\<\d\+[eE][+-]\=\d\+[jJ]\=\>/
+                   match /\<\d\+\.\%([eE][+-]\=\d\+\)\=[jJ]\=\%(\W\|$\)\@=/
+                   match /\%(^\|\W\)\@<=\d*\.\d\+\%([eE][+-]\=\d\+\)\=[jJ]\=\>/
+                   links to Number
+pythonBuiltin  xxx list locals all classmethod None abs cmp reduce ord hex object memoryview enumerate __debug__ compile str False True issubclass input hasattr frozenset slice callable sum filter range any long execfile min type sorted reload super complex xrange file ascii setattr unicode staticmethod basestring unichr float iter map globals max isinstance dict chr reversed buffer delattr __import__ oct dir eval raw_input hash getattr tuple id bin vars apply bytes repr pow print zip open NotImplemented intern round format bool help property coerce Ellipsis len int next exec set bytearray divmod
+                   links to Function
+pythonExceptions xxx OSError EnvironmentError UserWarning NameError ArithmeticError NotImplementedError ReferenceError BaseException LookupError ImportWarning OverflowError SystemExit IndentationError GeneratorExit Warning RuntimeError MemoryError WindowsError AssertionError UnicodeWarning KeyError TypeError TabError ImportError SyntaxWarning SyntaxError UnboundLocalError KeyboardInterrupt UnicodeDecodeError IOError Exception FutureWarning AttributeError UnicodeTranslateError VMSError EOFError FloatingPointError ValueError IndexError RuntimeWarning DeprecationWarning PendingDeprecationWarning UnicodeEncodeError StopIteration UnicodeError BytesWarning StandardError SystemError ZeroDivisionError BufferError
+                   links to Structure
+pythonDoctestValue xxx start=/^\s*\%(>>>\s\|\.\.\.\s\|"""\|'''\)\@!\S\+/ end=/$/  contained
+                   links to Define
+OperatorChars  xxx match #?\|+\|-\|\*\|;\|:\|,\|<\|>\|&\||\|!\|\~\|%\|=\|)\|(\|{\|}\|\.\|\[\|\]\|/\(/\|*\)\@!#
+Spell          cluster=NONE
+NoSpell        cluster=NONE
+