From e8b1af724b89ae14fa4c0676662166abd75c7f96 Mon Sep 17 00:00:00 2001
From: Shougo Matsushita <Shougo.Matsu@gmail.com>
Date: Sun, 11 May 2014 17:46:20 +0900
Subject: [PATCH] Improve snippets parser

---
 autoload/neosnippet/commands.vim            | 13 ++--
 autoload/neosnippet/parser.vim              | 59 ++++++++-------
 autoload/vital/_neosnippet.vim              | 61 +++++++++++++--
 autoload/vital/_neosnippet/Data/List.vim    | 12 +++
 autoload/vital/_neosnippet/Prelude.vim      | 24 ++----
 autoload/vital/_neosnippet/Process.vim      |  8 +-
 autoload/vital/_neosnippet/System/Cache.vim | 84 +++++++++++++++++++++
 autoload/vital/neosnippet.vital             |  3 +-
 doc/neosnippet.txt                          |  4 +-
 9 files changed, 199 insertions(+), 69 deletions(-)
 create mode 100644 autoload/vital/_neosnippet/System/Cache.vim

diff --git a/autoload/neosnippet/commands.vim b/autoload/neosnippet/commands.vim
index 796b96e..bf349a1 100644
--- a/autoload/neosnippet/commands.vim
+++ b/autoload/neosnippet/commands.vim
@@ -104,6 +104,7 @@ function! neosnippet#commands#_make_cache(filetype) "{{{
   if has_key(snippets, filetype)
     return
   endif
+  let snippets[filetype] = {}
 
   let path = join(neosnippet#helpers#get_snippets_directory(), ',')
   let snippets_files = []
@@ -117,19 +118,19 @@ function! neosnippet#commands#_make_cache(filetype) "{{{
     let snippets_files += split(globpath(path, glob), '\n')
   endfor
 
-  let snippet = {}
-  call map(reverse(s:get_list().uniq(snippets_files)),
-        \ "neosnippet#parser#_parse(snippet, v:val)")
-
   let snippets = neosnippet#variables#snippets()
-  let snippets[filetype] = snippet
+  for snippet_file in reverse(s:get_list().uniq(snippets_files))
+    let snippets[filetype] = extend(snippets[filetype],
+          \ neosnippet#parser#_parse(snippet_file))
+  endfor
 endfunction"}}}
 
 function! neosnippet#commands#_source(filename) "{{{
   call neosnippet#init#check()
 
   let neosnippet = neosnippet#variables#current_neosnippet()
-  call neosnippet#parser#_parse(neosnippet.snippets, a:filename)
+  let neosnippet.snippets = extend(neosnippet.snippets,
+        \ neosnippet#parser#_parse(a:filename))
 endfunction"}}}
 
 function! neosnippet#commands#_clear_markers() "{{{
diff --git a/autoload/neosnippet/parser.vim b/autoload/neosnippet/parser.vim
index 7fce584..9f34686 100644
--- a/autoload/neosnippet/parser.vim
+++ b/autoload/neosnippet/parser.vim
@@ -26,19 +26,25 @@
 let s:save_cpo = &cpo
 set cpo&vim
 
-function! neosnippet#parser#_parse(snippets, snippets_file) "{{{
-  let dup_check = {}
-  let snippet_dict = {}
+let s:Cache = neosnippet#util#get_vital().import('System.Cache')
 
-  let linenr = 1
-
-  if !filereadable(a:snippets_file)
+function! neosnippet#parser#_parse(snippet_file) "{{{
+  if !filereadable(a:snippet_file)
     call neosnippet#util#print_error(
-          \ printf('snippet file "%s" is not found.', a:snippets_file))
-    return a:snippets
+          \ printf('snippet file "%s" is not found.', a:snippet_file))
+    return {}
   endif
 
-  for line in readfile(a:snippets_file)
+  return s:parse(a:snippet_file)
+endfunction"}}}
+
+function! s:parse(snippet_file) "{{{
+  let dup_check = {}
+  let snippet_dict = {}
+  let linenr = 1
+  let snippets = {}
+
+  for line in readfile(a:snippet_file)
     if line =~ '^\h\w*.*\s$'
       " Delete spaces.
       let line = substitute(line, '\s\+$', '', '')
@@ -50,25 +56,26 @@ function! neosnippet#parser#_parse(snippets, snippets_file) "{{{
       " Include snippets.
       let filename = matchstr(line, '^include\s\+\zs.*$')
 
-      for snippets_file in split(globpath(join(
+      for snippet_file in split(globpath(join(
             \ neosnippet#helpers#get_snippets_directory(), ','),
             \ filename), '\n')
-        call neosnippet#parser#_parse(a:snippets, snippets_file)
+        let snippets = extend(snippets,
+              \ neosnippet#parser#_parse(snippets, snippet_file))
       endfor
     elseif line =~ '^delete\s'
       let name = matchstr(line, '^delete\s\+\zs.*$')
-      if name != '' && has_key(a:snippets, name)
-        call filter(a:snippets, 'v:val.real_name !=# name')
+      if name != '' && has_key(snippets, name)
+        call filter(snippets, 'v:val.real_name !=# name')
       endif
     elseif line =~ '^snippet\s'
       if !empty(snippet_dict)
         " Set previous snippet.
         call s:set_snippet_dict(snippet_dict,
-              \ a:snippets, dup_check, a:snippets_file)
+              \ snippets, dup_check, a:snippet_file)
       endif
 
       let snippet_dict = s:parse_snippet_name(
-            \ a:snippets_file, line, linenr, dup_check)
+            \ a:snippet_file, line, linenr, dup_check)
     elseif !empty(snippet_dict)
       if line =~ '^\s' || line == ''
         if snippet_dict.word == ''
@@ -77,10 +84,10 @@ function! neosnippet#parser#_parse(snippets, snippets_file) "{{{
         endif
 
         let snippet_dict.word .=
-                \ substitute(line, '^ *', '', '') . "\n"
+              \ substitute(line, '^ *', '', '') . "\n"
       else
         call s:add_snippet_attribute(
-              \ a:snippets_file, line, linenr, snippet_dict)
+              \ a:snippet_file, line, linenr, snippet_dict)
       endif
     endif
 
@@ -90,13 +97,13 @@ function! neosnippet#parser#_parse(snippets, snippets_file) "{{{
   if !empty(snippet_dict)
     " Set previous snippet.
     call s:set_snippet_dict(snippet_dict,
-          \ a:snippets, dup_check, a:snippets_file)
+          \ snippets, dup_check, a:snippet_file)
   endif
 
-  return a:snippets
+  return snippets
 endfunction"}}}
 
-function! s:parse_snippet_name(snippets_file, line, linenr, dup_check) "{{{
+function! s:parse_snippet_name(snippet_file, line, linenr, dup_check) "{{{
   " Initialize snippet dict.
   let snippet_dict = { 'word' : '', 'linenr' : a:linenr,
         \ 'options' : neosnippet#parser#_initialize_snippet_options() }
@@ -125,7 +132,7 @@ function! s:parse_snippet_name(snippets_file, line, linenr, dup_check) "{{{
     let dup = a:dup_check[snippet_dict.name]
     call neosnippet#util#print_error(printf(
           \ 'Warning: %s:%d is overriding `%s` from %s:%d',
-          \ a:snippets_file, a:linenr, snippet_dict.name,
+          \ a:snippet_file, a:linenr, snippet_dict.name,
           \ dup.action__path, dup.action__line))
     call neosnippet#util#print_error(printf(
           \ 'Please rename the snippet name or use `delete %s`.',
@@ -135,7 +142,7 @@ function! s:parse_snippet_name(snippets_file, line, linenr, dup_check) "{{{
   return snippet_dict
 endfunction"}}}
 
-function! s:add_snippet_attribute(snippets_file, line, linenr, snippet_dict) "{{{
+function! s:add_snippet_attribute(snippet_file, line, linenr, snippet_dict) "{{{
   " Allow overriding/setting of the description (abbr) of the snippet.
   " This will override what was set via the snippet line.
   if a:line =~ '^abbr\s'
@@ -161,7 +168,7 @@ function! s:add_snippet_attribute(snippets_file, line, linenr, snippet_dict) "{{
           \ '^options\s\+\zs.*$'), '[,[:space:]]\+')
       if !has_key(a:snippet_dict.options, option)
         call neosnippet#util#print_error(
-              \ printf('[neosnippet] %s:%d', a:snippets_file, a:linenr))
+              \ printf('[neosnippet] %s:%d', a:snippet_file, a:linenr))
         call neosnippet#util#print_error(
               \ printf('[neosnippet] Invalid option name : "%s"', option))
       else
@@ -170,20 +177,20 @@ function! s:add_snippet_attribute(snippets_file, line, linenr, snippet_dict) "{{
     endfor
   else
     call neosnippet#util#print_error(
-          \ printf('[neosnippet] %s:%d', a:snippets_file, a:linenr))
+          \ printf('[neosnippet] %s:%d', a:snippet_file, a:linenr))
     call neosnippet#util#print_error(
           \ printf('[neosnippet] Invalid syntax : "%s"', a:line))
   endif
 endfunction"}}}
 
-function! s:set_snippet_dict(snippet_dict, snippets, dup_check, snippets_file) "{{{
+function! s:set_snippet_dict(snippet_dict, snippets, dup_check, snippet_file) "{{{
   if empty(a:snippet_dict)
     return
   endif
 
   let action_pattern = '^snippet\s\+' . a:snippet_dict.name . '$'
   let snippet = neosnippet#parser#_initialize_snippet(
-        \ a:snippet_dict, a:snippets_file,
+        \ a:snippet_dict, a:snippet_file,
         \ a:snippet_dict.linenr, action_pattern,
         \ a:snippet_dict.name)
   let a:snippets[a:snippet_dict.name] = snippet
diff --git a/autoload/vital/_neosnippet.vim b/autoload/vital/_neosnippet.vim
index 6a0aac3..f8eff02 100644
--- a/autoload/vital/_neosnippet.vim
+++ b/autoload/vital/_neosnippet.vim
@@ -59,6 +59,19 @@ function! s:unload()
   let s:loaded = {}
 endfunction
 
+function! s:exists(name)
+  return s:_get_module_path(a:name) !=# ''
+endfunction
+
+function! s:search(pattern)
+  let target = substitute(a:pattern, '\.', '/', 'g')
+  let tailpath = printf('autoload/vital/%s/%s.vim', s:self_version, target)
+
+  let paths = s:_runtime_files(tailpath)
+  let modules = sort(map(paths, 's:_file2module(v:val)'))
+  return s:_uniq(modules)
+endfunction
+
 function! s:_import(name)
   if type(a:name) == type(0)
     return s:_build_module(a:name)
@@ -89,19 +102,16 @@ function! s:_get_module_path(name)
   if a:name ==# ''
     let tailpath = printf('autoload/vital/%s.vim', s:self_version)
   elseif a:name =~# '\v^\u\w*%(\.\u\w*)*$'
-    let target = '/' . substitute(a:name, '\W\+', '/', 'g')
-    let tailpath = printf('autoload/vital/%s%s.vim', s:self_version, target)
+    let target = substitute(a:name, '\W\+', '/', 'g')
+    let tailpath = printf('autoload/vital/%s/%s.vim', s:self_version, target)
   else
     throw 'vital: Invalid module name: ' . a:name
   endif
 
-  if s:globpath_third_arg
-    let paths = split(globpath(&runtimepath, tailpath, 1), "\n")
-  else
-    let paths = split(globpath(&runtimepath, tailpath), "\n")
-  endif
+  let paths = s:_runtime_files(tailpath)
   call filter(paths, 'filereadable(v:val)')
-  return s:_unify_path(get(paths, 0, ''))
+  let path = get(paths, 0, '')
+  return path !=# '' ? s:_unify_path(path) : ''
 endfunction
 
 function! s:_scripts()
@@ -116,6 +126,12 @@ function! s:_scripts()
   return scripts
 endfunction
 
+function! s:_file2module(file)
+  let filename = s:_unify_path(a:file)
+  let tail = matchstr(filename, 'autoload/vital/_\w\+/\zs.*\ze\.vim$')
+  return join(split(tail, '[\\/]\+'), '.')
+endfunction
+
 if filereadable(expand('<sfile>:r') . '.VIM')
   function! s:_unify_path(path)
     " Note: On windows, vim can't expand path names from 8.3 formats.
@@ -130,6 +146,16 @@ else
   endfunction
 endif
 
+if s:globpath_third_arg
+  function! s:_runtime_files(path)
+    return split(globpath(&runtimepath, a:path, 1), "\n")
+  endfunction
+else
+  function! s:_runtime_files(path)
+    return split(globpath(&runtimepath, a:path), "\n")
+  endfunction
+endif
+
 " Copy from System.Filepath
 if has('win16') || has('win32') || has('win64')
   function! s:_is_absolute_path(path)
@@ -188,6 +214,25 @@ else
   endfunction
 endif
 
+if exists('*uniq')
+  function! s:_uniq(list)
+    return uniq(a:list)
+  endfunction
+else
+  function! s:_uniq(list)
+    let i = len(a:list) - 1
+    while 0 < i
+      if a:list[i] ==# a:list[i - 1]
+        call remove(a:list, i)
+        let i -= 2
+      else
+        let i -= 1
+      endif
+    endwhile
+    return a:list
+  endfunction
+endif
+
 function! s:_redir(cmd)
   let [save_verbose, save_verbosefile] = [&verbose, &verbosefile]
   set verbose=0 verbosefile=
diff --git a/autoload/vital/_neosnippet/Data/List.vim b/autoload/vital/_neosnippet/Data/List.vim
index cdbd49b..29e373e 100644
--- a/autoload/vital/_neosnippet/Data/List.vim
+++ b/autoload/vital/_neosnippet/Data/List.vim
@@ -198,6 +198,18 @@ function! s:or(xs)
   return s:any('v:val', a:xs)
 endfunction
 
+function! s:map_accum(expr, xs, init)
+  let memo = []
+  let init = a:init
+  for x in a:xs
+    let expr = substitute(a:expr, 'v:memo', init, 'g')
+    let expr = substitute(expr, 'v:val', x, 'g')
+    let [tmp, init] = eval(expr)
+    call add(memo, tmp)
+  endfor
+  return memo
+endfunction
+
 " similar to Haskell's Prelude.foldl
 function! s:foldl(f, init, xs)
   let memo = a:init
diff --git a/autoload/vital/_neosnippet/Prelude.vim b/autoload/vital/_neosnippet/Prelude.vim
index 9a076bc..e9191ae 100644
--- a/autoload/vital/_neosnippet/Prelude.vim
+++ b/autoload/vital/_neosnippet/Prelude.vim
@@ -1,9 +1,6 @@
 let s:save_cpo = &cpo
 set cpo&vim
 
-" glob() wrapper which returns List
-" and 'wildignore' does not affect
-" this function's return value.
 if v:version ># 703 ||
 \  (v:version is 703 && has('patch465'))
   function! s:glob(expr)
@@ -16,9 +13,6 @@ else
   endfunction
 endif
 
-" globpath() wrapper which returns List
-" and 'suffixes' and 'wildignore' does not affect
-" this function's return value.
 function! s:globpath(path, expr)
   let R = globpath(a:path, a:expr, 1)
   return split(R, '\n')
@@ -215,19 +209,9 @@ function! s:is_unix()
   return s:is_unix
 endfunction
 
-function! s:_deprecated(fname, newname)
-  echomsg printf("Vital.Prelude.%s is deprecated! Please use %s instead.",
-        \ a:fname, a:newname)
-endfunction
-
-function! s:print_error(message)
-  call s:_deprecated('print_error', 'Vital.Vim.Message.error')
-
-  echohl ErrorMsg
-  for m in split(a:message, "\n")
-    echomsg m
-  endfor
-  echohl None
+function! s:_deprecated2(fname)
+  echomsg printf("Vital.Prelude.%s is deprecated!",
+        \ a:fname)
 endfunction
 
 function! s:smart_execute_command(action, word)
@@ -277,6 +261,8 @@ function! s:set_default(var, val)
 endfunction
 
 function! s:set_dictionary_helper(variable, keys, pattern)
+  call s:_deprecated2('set_dictionary_helper')
+
   for key in split(a:keys, '\s*,\s*')
     if !has_key(a:variable, key)
       let a:variable[key] = a:pattern
diff --git a/autoload/vital/_neosnippet/Process.vim b/autoload/vital/_neosnippet/Process.vim
index f2dc640..cdeb875 100644
--- a/autoload/vital/_neosnippet/Process.vim
+++ b/autoload/vital/_neosnippet/Process.vim
@@ -119,13 +119,7 @@ function! s:system(str, ...)
     let args += [input] + rest
   endif
 
-  if use_vimproc
-    " vimproc's parser seems to treat # as a comment
-    let args[0] = escape(args[0], '#')
-    let funcname = 'vimproc#system'
-  else
-    let funcname = 'system'
-  endif
+  let funcname = use_vimproc ? 'vimproc#system' : 'system'
   let output = call(funcname, args)
   let output = s:iconv(output, 'char', &encoding)
 
diff --git a/autoload/vital/_neosnippet/System/Cache.vim b/autoload/vital/_neosnippet/System/Cache.vim
new file mode 100644
index 0000000..54e3405
--- /dev/null
+++ b/autoload/vital/_neosnippet/System/Cache.vim
@@ -0,0 +1,84 @@
+" Utilities for output cache.
+
+let s:save_cpo = &cpo
+set cpo&vim
+
+function! s:getfilename(cache_dir, filename)
+  return s:_encode_name(a:cache_dir, a:filename)
+endfunction
+
+function! s:filereadable(cache_dir, filename)
+  let cache_name = s:_encode_name(a:cache_dir, a:filename)
+  return filereadable(cache_name)
+endfunction
+
+function! s:readfile(cache_dir, filename)
+  let cache_name = s:_encode_name(a:cache_dir, a:filename)
+  return filereadable(cache_name) ? readfile(cache_name) : []
+endfunction
+
+function! s:writefile(cache_dir, filename, list)
+  let cache_name = s:_encode_name(a:cache_dir, a:filename)
+
+  call writefile(a:list, cache_name)
+endfunction
+
+function! s:delete(cache_dir, filename)
+  echoerr 'System.Cache.delete() is obsolete. Use its deletefile() instead.'
+  return call('s:deletefile', a:cache_dir, a:filename)
+endfunction
+
+function! s:deletefile(cache_dir, filename)
+  let cache_name = s:_encode_name(a:cache_dir, a:filename)
+  return delete(cache_name)
+endfunction
+
+function! s:_encode_name(cache_dir, filename)
+  " Check cache directory.
+  if !isdirectory(a:cache_dir)
+    call mkdir(a:cache_dir, 'p')
+  endif
+  let cache_dir = a:cache_dir
+  if cache_dir !~ '/$'
+    let cache_dir .= '/'
+  endif
+
+  return cache_dir . s:_create_hash(cache_dir, a:filename)
+endfunction
+
+function! s:check_old_cache(cache_dir, filename)
+  " Check old cache file.
+  let cache_name = s:_encode_name(a:cache_dir, a:filename)
+  let ret = getftime(cache_name) == -1
+        \ || getftime(cache_name) <= getftime(a:filename)
+  if ret && filereadable(cache_name)
+    " Delete old cache.
+    call delete(cache_name)
+  endif
+
+  return ret
+endfunction
+
+function! s:_create_hash(dir, str)
+  if len(a:dir) + len(a:str) < 150
+    let hash = substitute(substitute(
+          \ a:str, ':', '=-', 'g'), '[/\\]', '=+', 'g')
+  elseif exists('*sha256')
+    let hash = sha256(a:str)
+  else
+    " Use simple hash.
+    let sum = 0
+    for i in range(len(a:str))
+      let sum += char2nr(a:str[i]) * (i + 1)
+    endfor
+
+    let hash = printf('%x', sum)
+  endif
+
+  return hash
+endfunction
+
+let &cpo = s:save_cpo
+unlet s:save_cpo
+
+" vim:set et ts=2 sts=2 sw=2 tw=0:
diff --git a/autoload/vital/neosnippet.vital b/autoload/vital/neosnippet.vital
index a63048f..418bf1d 100644
--- a/autoload/vital/neosnippet.vital
+++ b/autoload/vital/neosnippet.vital
@@ -1,6 +1,7 @@
 neosnippet
-450727f
+d3554a5
 
 Prelude
 Data.List
 Process
+System.Cache
diff --git a/doc/neosnippet.txt b/doc/neosnippet.txt
index f5daab8..a42adee 100755
--- a/doc/neosnippet.txt
+++ b/doc/neosnippet.txt
@@ -612,8 +612,8 @@ Or if you want to include a whole directory with file type snippets.
 <
 
 If you include snippet files it can happen that the same snippet name is used
-multiple times in different snippet files. Neosnippet produces a warning if it
-detects this. If you want to overwrite a snippet explicitly, please use:
+multiple times in snippet files. Neosnippet produces a warning if it detects
+this. If you want to overwrite a snippet explicitly, please use:
 
 >
 	delete snippets_name