arc lint: add support for PyLint

Summary:
Provides a lint class as a wrapper around the external project PyLint.
This exposes some arc config variables to control the behavior:

lint.pylint.prefix - non-standard installation location of pylint
lint.pylint.logilab_astng.prefix - non-standard installation location
  of logilab-astng, a dependency of pylint
lint.pylint.logilab_common.prefix - non-standard installation location
  of logilab-common, a dependency of pylint
lint.pylint.codes.{error,warning,advice} - regexes matching against
  PyLint message codes which should trigger arc errors/warnings/advice
lint.pylint.options - options to pass PyLint

Test Plan:
used to lint python code

Reviewed By: epriestley
Reviewers: epriestley, jungejason
CC: aran, epriestley
Differential Revision: 343

src/__phutil_library_map__.php

@@ -62,6 +62,7 @@ phutil_register_library_map(array(
     'ArcanistPhutilTestCase' => 'unit/engine/phutil/testcase',
     'ArcanistPhutilTestTerminatedException' => 'unit/engine/phutil/testcase/exception',
     'ArcanistPyFlakesLinter' => 'lint/linter/pyflakes',
+    'ArcanistPyLintLinter' => 'lint/linter/pylint',
     'ArcanistRepositoryAPI' => 'repository/api/base',
     'ArcanistShellCompleteWorkflow' => 'workflow/shell-complete',
     'ArcanistSubversionAPI' => 'repository/api/subversion',
@@ -115,6 +116,7 @@ phutil_register_library_map(array(
     'ArcanistPatchWorkflow' => 'ArcanistBaseWorkflow',
     'ArcanistPhutilModuleLinter' => 'ArcanistLinter',
     'ArcanistPyFlakesLinter' => 'ArcanistLinter',
+    'ArcanistPyLintLinter' => 'ArcanistLinter',
     'ArcanistShellCompleteWorkflow' => 'ArcanistBaseWorkflow',
     'ArcanistSubversionAPI' => 'ArcanistRepositoryAPI',
     'ArcanistSvnHookPreCommitWorkflow' => 'ArcanistBaseWorkflow',

src/lint/linter/pylint/ArcanistPyLintLinter.php

@@ -0,0 +1,168 @@
+<?php
+
+/*
+ * Copyright 2011 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Uses "PyLint" to detect various errors in Python code.
+ *
+ * @group linter
+ */
+class ArcanistPyLintLinter extends ArcanistLinter {
+
+  private function getMessageCodeSeverity($code) {
+    // The config file defines how PyLint message codes translate to
+    // arcanist severities.  The config options provide regex's to
+    // match against the message codes generated by PyLint.  Severity's
+    // are matched in the order of errors, warnings, then advice.
+    // The first severity that matches, in that order, is returned.
+    $working_copy = $this->getEngine()->getWorkingCopy();
+    $code_map = array(
+      ArcanistLintSeverity::SEVERITY_ERROR =>
+          $working_copy->getConfig('lint.pylint.codes.error'),
+      ArcanistLintSeverity::SEVERITY_WARNING =>
+          $working_copy->getConfig('lint.pylint.codes.warning'),
+      ArcanistLintSeverity::SEVERITY_ADVICE =>
+          $working_copy->getConfig('lint.pylint.codes.advice'),
+    );
+
+    foreach ($code_map as $sev => $codes) {
+      if ($codes !== null) {
+        foreach ($codes as $code_re) {
+          if (preg_match("/{$code_re}/", $code)) {
+            return $sev;
+          }
+        }
+      }
+    }
+
+    // If the message code doesn't match any of the provided regex's,
+    // then just disable it.
+    return ArcanistLintSeverity::SEVERITY_DISABLED;
+  }
+
+  private function getPyLintPath() {
+    $pylint_bin = "pylint";
+
+    // Use the PyLint prefix specified in the config file
+    $working_copy = $this->getEngine()->getWorkingCopy();
+    $prefix = $working_copy->getConfig('lint.pylint.prefix');
+    if ($prefix !== null) {
+      $pylint_bin = $prefix."/bin/".$pylint_bin;
+    }
+
+    return $pylint_bin;
+  }
+
+  private function getPyLintPythonPath() {
+    // Get non-default install locations for pylint and its dependencies
+    // libraries.
+    $working_copy = $this->getEngine()->getWorkingCopy();
+    $prefixes = array(
+      $working_copy->getConfig('lint.pylint.prefix'),
+      $working_copy->getConfig('lint.pylint.logilab_astng.prefix'),
+      $working_copy->getConfig('lint.pylint.logilab_common.prefix'),
+    );
+
+    // Add the libraries to the python search path
+    $python_path = array();
+    foreach ($prefixes as $prefix) {
+      if ($prefix !== null) {
+        $python_path[] = $prefix.'/lib/python2.6/site-packages';
+      }
+    }
+
+    $python_path[] = '';
+    return implode(":", $python_path);
+  }
+
+  private function getPyLintOptions() {
+    // Options to pass the PyLint
+    // - '-rn': don't print lint report/summary at end
+    // - '-iy': show message codes for lint warnings/errors
+    $options = array('-rn',  '-iy');
+
+    // Add any options defined in the config file for PyLint
+    $working_copy = $this->getEngine()->getWorkingCopy();
+    $config_options = $working_copy->getConfig('lint.pylint.options');
+    if ($config_options !== null) {
+      $options += $config_options;
+    }
+
+    return implode(" ", $options);
+  }
+
+  public function willLintPaths(array $paths) {
+    return;
+  }
+
+  public function getLinterName() {
+    return 'PyLint';
+  }
+
+  public function getLintSeverityMap() {
+    return array();
+  }
+
+  public function getLintNameMap() {
+    return array();
+  }
+
+  public function lintPath($path) {
+    $pylint_bin = $this->getPyLintPath();
+    $python_path = $this->getPyLintPythonPath();
+    $options = $this->getPyLintOptions();
+    $path_on_disk = $this->getEngine()->getFilePathOnDisk($path);
+
+    try {
+      list($stdout, $_) = execx(
+          "/usr/bin/env PYTHONPATH=%s\$PYTHONPATH ".
+            "{$pylint_bin} {$options} {$path_on_disk}",
+          $python_path);
+    } catch (CommandException $e) {
+      // PyLint will return a non-zero exit code if warnings/errors are found.
+      // Therefore we detect command failure by checking that the stderr is
+      // some non-expected value.
+      if ($e->getStderr() !== "No config file found, ".
+                              "using default configuration\n") {
+        throw $e;
+      }
+
+      $stdout = $e->getStdout();
+    }
+
+    $lines = explode("\n", $stdout);
+    $messages = array();
+    foreach ($lines as $line) {
+      $matches = null;
+      if (!preg_match('/([A-Z]\d+): *(\d+): *(.*)$/', $line, $matches)) {
+        continue;
+      }
+      foreach ($matches as $key => $match) {
+        $matches[$key] = trim($match);
+      }
+      $message = new ArcanistLintMessage();
+      $message->setPath($path);
+      $message->setLine($matches[2]);
+      $message->setCode($matches[1]);
+      $message->setName($this->getLinterName()." ".$matches[1]);
+      $message->setDescription($matches[3]);
+      $message->setSeverity($this->getMessageCodeSeverity($matches[1]));
+      $this->addLintMessage($message);
+    }
+  }
+
+}

src/lint/linter/pylint/__init__.php

@@ -0,0 +1,16 @@
+<?php
+/**
+ * This file is automatically generated. Lint this module to rebuild it.
+ * @generated
+ */
+
+
+
+phutil_require_module('arcanist', 'lint/linter/base');
+phutil_require_module('arcanist', 'lint/message');
+phutil_require_module('arcanist', 'lint/severity');
+
+phutil_require_module('phutil', 'future/exec');
+
+
+phutil_require_source('ArcanistPyLintLinter.php');