M .hgsubstate +1 -1
@@ 1,4 1,4 @@
-6eb7692bbb52b9dec7ff18118df6f7d64d1d4c9e library
+d8f68e4052379d8ced50a792a07b06942926c9c5 library
19292d513d74939f058a037bbc5ea0ccd6acbc98 src/lithe
c3e562558279e535531dd37f3c2b2770e49000d9 src/pad
7987a84e315b3604f51f8f42a512f6b6bb5a1e6f src/paf
M CMakeLists.txt +2 -0
@@ 74,6 74,7 @@ set( KRONOS_CORE_SOURCES
"src/k3/Native.cpp"
"src/k3/NativeVector.cpp"
"src/k3/Parser.cpp"
+ "src/k3/CodeRepository.cpp"
"src/k3/Reactive.cpp"
"src/k3/RegionNode.cpp"
"src/k3/Stateful.cpp"
@@ 187,6 188,7 @@ source_group( "Graph" FILES
source_group( "Repository" FILES
"src/k3/Parser.cpp"
"src/k3/Parser.h"
+ "src/k3/CodeRepository.cpp"
"src/k3/SourceFileMap.h")
source_group( "Transforms" FILES
M src/common/Err.h +1 -0
@@ 4,6 4,7 @@
#include "Errors.h"
#define LET_ERR(sym, expr) auto sym ## _opt = expr; if (sym ## _opt.err) return std::move(sym ##_opt.err); auto sym = *sym ## _opt;
+#define CHECK_ERR(expr) { auto ___check = expr; if (___check.err) return std::move(___check.err); }
namespace K3 {
template <typename TValue> struct Err {
M src/common/Platform.cpp +26 -6
@@ 11,6 11,7 @@
#include <sys/stat.h>
#include <unistd.h>
#include <stdlib.h>
+#include <limits.h>
#include <libgen.h>
#include <sys/stat.h>
#include <string>
@@ 87,6 88,13 @@ std::string GetParentPath(std::string fi
return dirname((char*)filePath.data());
}
+std::string GetCanonicalAbsolutePath(std::string path) {
+ char fullPath[PATH_MAX] = { 0 };
+ auto rp = realpath(path.c_str(), fullPath);
+ if (rp) path = rp;
+ return path;
+}
+
std::string GetProcessID() {
return std::to_string(getpid());
}
@@ 134,11 142,23 @@ std::wstring utf8filename(const char* st
return cvt.from_bytes(str);
}
+std::string GetCanonicalAbsolutePath(std::string path) {
+ wchar_t full[_MAX_PATH ] = { 0 };
+ for (auto& p : path) {
+ if (p == '/') p = '\\';
+ }
+ auto wpath = utf8filename(path);
+ if (_wfullpath(full, wpath.data(), _MAX_PATH) != nullptr) {
+ path = encode_utf8(std::wstring(full));
+ }
+ return path;
+}
+
std::string GetParentPath(std::string filePath) {
- char drv[_MAX_DRIVE];
- char dir[_MAX_DIR];
- _splitpath(filePath.c_str(), drv, dir, nullptr, nullptr);
- return std::string(drv) + dir;
+ wchar_t drv[_MAX_DRIVE];
+ wchar_t dir[_MAX_DIR];
+ _wsplitpath(utf8filename(filePath).c_str(), drv, dir, nullptr, nullptr);
+ return encode_utf8(std::wstring(drv) + dir);
}
std::string GetProcessFileName( ) {
@@ 199,8 219,8 @@ time_t GetFileLastModified(const std::st
_wstat(utf8filename(str).c_str(),&stat);
return stat.st_mtime;
}
-#endif
-
+#endif
+
std::string GetConfigPath() {
if (getenv("XDG_CONFIG_HOME")) return getenv("XDG_CONFIG_HOME");
return GetUserPath();
M src/common/PlatformUtils.h +7 -6
@@ 21,13 21,13 @@ static const char* utf8filename(const ch
#endif
-template <typename TO, typename FROM> static TO check_cast(FROM f) {
+template <typename TO, typename FROM> static TO check_cast(FROM f) {
#ifndef NDEBUG
auto m = std::numeric_limits<TO>::min();
auto x = std::numeric_limits<TO>::max();
assert((f >= 0 || f >= m) && "sign lost in cast");
assert((f < 0 || f <= x) && "truncated cast");
- assert(static_cast<FROM>(static_cast<TO>(f)) == f && "cast loses precision");
+ assert(static_cast<FROM>(static_cast<TO>(f)) == f && "cast loses precision");
#endif
return static_cast<TO>(f);
}
@@ 38,10 38,11 @@ std::string GetMachineName( );
time_t GetFileLastModified(const std::string& filename);
std::string GetUserPath();
-std::string GetSharedPath();
-std::string GetCachePath();
-std::string GetConfigPath();
-
+std::string GetSharedPath();
+std::string GetCachePath();
+std::string GetConfigPath();
+
std::string GetParentPath(std::string pathToFile);
+std::string GetCanonicalAbsolutePath(std::string path);
std::string encode_utf8(const std::wstring& w);
M src/driver/LanguageServer.cpp +1 -1
@@ 792,7 792,7 @@ namespace Kronos {
}
};
- bbClient = std::make_unique<Packages::DefaultClient>(repository, repositoryVersion);
+ bbClient = std::make_unique<Packages::DefaultClient>();
auto endpoint = std::make_shared<LangSrvEndpoint>();
InjectMembers(*endpoint, std::make_shared<DocumentContext::ServerInstance>());
return endpoint;
M src/driver/kc.cpp +3 -4
@@ 67,9 67,8 @@ int main(int n, const char *carg[]) {
stringstream log;
Context cx;
- const char *repo = getenv("KRONOS_CORE_REPOSITORY");
- const char *repoVersion = getenv("KRONOS_CORE_REPOSITORY_VERSION");
- Packages::DefaultClient bbClient(repo ? repo : KRONOS_CORE_LIBRARY_REPOSITORY, repoVersion ? repoVersion : KRONOS_CORE_LIBRARY_VERSION);
+
+ Packages::DefaultClient bbClient;
try {
std::list<const char*> args;
@@ 96,7 95,7 @@ int main(int n, const char *carg[]) {
if ((args.empty( ) && CL::input().empty( )) || CL::input() == "-") {
stringstream readStdin;
readStdin << cin.rdbuf( );
- myContext.ImportBuffer("#stdin", readStdin.str( ));
+ myContext.ImportBuffer(readStdin.str( ));
}
if (CL::input() != "-" && CL::input().size( )) myContext.ImportFile(CL::input());
M src/driver/krepl.cpp +7 -8
@@ 8,7 8,6 @@
#include "CmdLineOpts.h"
#include "config/system.h"
-#include "config/corelib.h"
#include "driver/package.h"
#include "kronos.h"
#include "friends.h"
@@ 42,18 41,13 @@ void FormatErrors(const char* xml, std::
int krepl_main(CmdLine::IRegistry& CLOpts, Kronos::IO::IConfiguringHierarchy* io, int argn, const char *carg[] ) {
using namespace Kronos;
-
- const char *repo = getenv("KRONOS_CORE_REPOSITORY");
- const char *repoVersion = getenv("KRONOS_CORE_REPOSITORY_VERSION");
- if (!repo) repo = KRONOS_CORE_LIBRARY_REPOSITORY;
- if (!repoVersion) repoVersion = KRONOS_CORE_LIBRARY_VERSION;
int err = 0;
CL::SetRegistry(CLOpts);
try {
- Packages::DefaultClient bbClient(repo, repoVersion);
+ Packages::DefaultClient bbClient;
std::list<const char*> args;
if (!cx) Kronos::AddBackendCmdLineOpts(CLOpts);
@@ 81,11 75,16 @@ int krepl_main(CmdLine::IRegistry& CLOpt
if (CL::interactive() && repl_args.empty()) {
std::stringstream banner;
+ std::string repo, repoVersion;
+ cx.GetCoreLibrary(repo, repoVersion);
banner << "\"Welcome to KREPL " << KRONOS_PACKAGE_VERSION << " [" << repo << " " << repoVersion << "]\"";
repl_args.emplace_back(banner.str());
}
- REPL::JiT::Compiler compiler{ cx, bbClient.Resolve("", "VM.k", "") };
+ std::string coreRepo, coreVersion;
+ cx.GetCoreLibrary(coreRepo, coreVersion);
+
+ REPL::JiT::Compiler compiler{ cx, bbClient.Resolve(coreRepo, "VM.k", coreVersion) };
REPL::CompilerConfigurer cfg{ compiler, io };
compiler.SetLogFormatter([](Context& cx, const std::string& xml, std::ostream& fmt) {
M src/driver/krpc.cpp +1 -7
@@ 31,16 31,11 @@ Kronos::Context cx;
int main( int argn, const char *carg[] ) {
using namespace Kronos;
- const char *repo = getenv("KRONOS_CORE_REPOSITORY");
- const char *repoVersion = getenv("KRONOS_CORE_REPOSITORY_VERSION");
- if (!repo) repo = KRONOS_CORE_LIBRARY_REPOSITORY;
- if (!repoVersion) repoVersion = KRONOS_CORE_LIBRARY_VERSION;
-
std::list<const char*> args;
Kronos::AddBackendCmdLineOpts(CmdLine::Registry());
for (int i(1);i < argn;++i) args.emplace_back(carg[i]);
- Packages::DefaultClient bbClient(repo ? repo : KRONOS_CORE_LIBRARY_REPOSITORY, repoVersion ? repoVersion : KRONOS_CORE_LIBRARY_VERSION);
+ Packages::DefaultClient bbClient;
int err = 0;
@@ 50,7 45,6 @@ int main( int argn, const char *carg[] )
}
std::clog << "KRPC; Kronos " KRONOS_PACKAGE_VERSION " RPC Endpoint \n(c) 2018 Vesa Norilo, University of the Arts Helsinki\n\n";
- std::clog << "[" << repo << " " << repoVersion << "]\n";
cx = CreateContext(Packages::CloudClient::ResolverCallback, &bbClient);
M src/driver/krpcsrv.cpp +1 -3
@@ 93,10 93,8 @@ int main(int argn, const char* carg[]) {
using namespace Kronos;
using namespace Sxx;
- const char *repo = getenv("KRONOS_CORE_REPOSITORY");
- const char *repoVersion = getenv("KRONOS_CORE_REPOSITORY_VERSION");
- Packages::DefaultClient bbClient(repo ? repo : KRONOS_CORE_LIBRARY_REPOSITORY, repoVersion ? repoVersion : KRONOS_CORE_LIBRARY_VERSION);
+ Packages::DefaultClient bbClient;
AssetManager = &bbClient;
int resultCode = 0;
M src/driver/ktests.cpp +1 -9
@@ 205,11 205,6 @@ int main(int argc, const char* arg[]) {
CLOpts::dbauth() = "Authorization: " + CLOpts::dbauth();
}
- const char *repo = getenv("KRONOS_CORE_REPOSITORY");
- const char *repoVersion = getenv("KRONOS_CORE_REPOSITORY_VERSION");
- if (!repo) repo = KRONOS_CORE_LIBRARY_REPOSITORY;
- if (!repoVersion) repoVersion = KRONOS_CORE_LIBRARY_VERSION;
-
std::clog << "Testing [" << CLOpts::package() << " " << CLOpts::package_version() << "]\n";
if (CLOpts::package() == KRONOS_CORE_LIBRARY_REPOSITORY) {
// if we are testing core, use it as the default package
@@ 218,11 213,9 @@ int main(int argc, const char* arg[]) {
CLOpts::package_version = KRONOS_CORE_LIBRARY_VERSION;
}
- repoVersion = CLOpts::package_version().c_str();
static std::string coreVersion = "KRONOS_CORE_REPOSITORY_VERSION=" + CLOpts::package_version();
putenv((char*)coreVersion.c_str());
} else {
- std::clog << "Core [" << repo << " " << repoVersion << "]\n";
if (CLOpts::package_version().empty()) {
// test the tip
CLOpts::package_version = "tip~";
@@ 232,7 225,7 @@ int main(int argc, const char* arg[]) {
std::clog << "\n";
- Packages::DefaultClient bbClient(repo, repoVersion);
+ Packages::DefaultClient bbClient;
auto testDataFilePath = bbClient.Resolve(CLOpts::package(), "tests.json", CLOpts::package_version());
std::ifstream testDataStream(testDataFilePath);
@@ 248,7 241,6 @@ int main(int argc, const char* arg[]) {
auto ri = GetRunInfo();
ri["package"] = picojson::array{ CLOpts::package(), SplitSemVer(CLOpts::package_version()) };
- ri["corelib"] = picojson::array{ bbClient.GetCoreRepository(), SplitSemVer(bbClient.GetCoreVersion()) };
if (getenv("BUILD_COMMIT_HASH")) ri["commit"] = getenv("BUILD_COMMIT_HASH");
if (CLOpts::build_hash().size()) ri["commit"] = CLOpts::build_hash();
M src/driver/kwasm.cpp +9 -2
@@ 146,7 146,14 @@ public:
cx.ImportFile(Resolve(KRONOS_CORE_LIBRARY_REPOSITORY, "VM.k", KRONOS_CORE_LIBRARY_VERSION)); LogErr();
cx.SetAssetLinker(WasmAssetLinker, this); LogErr();
cx.RegisterSpecializationCallback("rpc-monitor", [](void *user, int diags, const IType* ty, int64_t tyUid) {
- if (!diags) return;
+ if (!diags) {
+ auto self = (WasmCompiler*)user;
+ if (self->specializationMonitor.as<bool>()) {
+ self->specializationMonitor("{\"jsonrpc\":2.0,\"method\":\"rpc-size\",\"params\": "
+ + std::to_string(ty->SizeOf()) + "}");
+ }
+ return;
+ }
std::stringstream tyStr;
StreamBuf sbuf(tyStr.rdbuf());
@@ 160,8 167,8 @@ public:
TypeString(monitor, tyStr.str().c_str());
monitor << "}}";
+ std::clog << monitor.str() << std::endl;
auto self = (WasmCompiler*)user;
- std::clog << monitor.str() << std::endl;
if (self->specializationMonitor.as<bool>()) self->specializationMonitor(monitor.str());
}, this);
LogErr();
M src/driver/package.cpp +1 -14
@@ 253,7 253,7 @@ namespace Packages {
return ((CloudClient*)self)->Resolve(pack, file, ver);
}
- CloudClient::CloudClient(const std::string& dr, const std::string &dver) :defaultRepository(dr), defaultVersion(dver) {
+ CloudClient::CloudClient() {
using namespace std::string_literals;
fileSystemLocation = GetCachePath() + "/";
@@ 304,13 304,6 @@ namespace Packages {
}
const char* CloudClient::Resolve(std::string pack, std::string file, std::string version) {
- if (pack == "#core") {
- pack = defaultRepository;
- version = defaultVersion;
- } else {
- if (pack.empty()) pack = defaultRepository;
- if (version.empty()) version = defaultVersion;
- }
auto path = GetLocalFilePath(pack, version);
auto file_in = path + file;
if (!DoesFileExist(file_in)) {
@@ 368,9 361,6 @@ namespace Packages {
}
}
- GitHubApi::GitHubApi(const std::string& defaultRepo, const std::string& defaultVer)
- :CloudClient(defaultRepo, defaultVer) {}
-
std::string GitHubApi::GetCommitHash(const std::string& pack, const std::string& version) const {
std::string sha = "";
std::lock_guard<std::recursive_mutex> lg{ lock };
@@ 420,9 410,6 @@ namespace Packages {
}
}
-
- BitBucketCloud::BitBucketCloud(const std::string & defaultRepo, const std::string & defaultVer):CloudClient(defaultRepo, defaultVer) {}
-
static inline uint64_t mul_128hi(uint64_t x, uint64_t y) {
const uint64_t m32 = 0xffffffff;
uint64_t x0 = x & m32, y0 = y & m32, x1 = x >> 32, y1 = y >> 32;
M src/driver/package.h +4 -13
@@ 8,8 8,6 @@
namespace Packages {
class CloudClient {
- std::string defaultRepository;
- std::string defaultVersion;
std::string fileSystemLocation;
std::unordered_set<std::string> stringPool;
protected:
@@ 21,17 19,10 @@ namespace Packages {
void AddPackageVersion(const std::string& package, const std::string& version, const std::string& hash) const;
const std::string& GetFileSystemLocation() const { return fileSystemLocation; }
public:
- CloudClient(const std::string& defaultRepository = "", const std::string& defaultVersion = "");
+ CloudClient();
static const char* ResolverCallback(const char *package, const char *file, const char *version, void *self);
- const char* Resolve(std::string package, std::string file, std::string version);
+ const char* Resolve(std::string package, std::string file, std::string version);
std::string GetLocalFilePath(std::string package, std::string version) const;
- const char *GetCoreRepository() const {
- return defaultRepository.c_str();
- }
-
- const char *GetCoreVersion() const {
- return defaultVersion.c_str();
- }
const picojson::object& GetCache() const {
return cacheData;
@@ 44,13 35,13 @@ namespace Packages {
std::string GetCommitHash(const std::string & pack, const std::string & version) const override;
void Obtain(std::string package, std::string version, std::string file, std::string dstPath) override;
public:
- BitBucketCloud(const std::string &defaultRepo, const std::string& defaultVer);
+ BitBucketCloud() { };
};
class GitHubApi : public CloudClient {
void Obtain(std::string package, std::string version, std::string file, std::string dstPath) override;
public:
- GitHubApi(const std::string &defaultRepo, const std::string& defaultVer);
+ GitHubApi() { };
std::string GetCommitHash(const std::string & pack, const std::string & version) const;
};
A => src/k3/CodeRepository.cpp +637 -0
@@ 0,0 1,637 @@
+#include "Parser.h"
+#include "Invariant.h"
+#include "lithe/grammar/kronos.h"
+#include "config/corelib.h"
+#include "TLS.h"
+#include "Evaluate.h"
+#include "TypeAlgebra.h"
+#include "Invariant.h"
+#include "LibraryRef.h"
+#include "common/PlatformUtils.h"
+
+#include <fstream>
+#include <sstream>
+#include <regex>
+
+#define Repository Repository2
+
+namespace K3 {
+ namespace Parser {
+ static const char* DefinitionName[] = {
+ "undefined",
+ "variable",
+ "function"
+ };
+
+ Form Form::Fn(CGRef expr) {
+ Form f;
+ f.body = expr;
+ f.mode = Mode::Function;
+ return f;
+ }
+
+ Form Form::Val(CGRef expr) {
+ Form f;
+ f.body = expr;
+ f.mode = Mode::Macro;
+ return f;
+ }
+
+ ParserError Repository::RedefinitionError(std::string qn,
+ const char* newPoint, Form::Mode newMode,
+ const char* oldPoint, Form::Mode oldMode) {
+ using namespace std::string_literals;
+ std::string URI, show;
+ int line, column;
+
+ GetPosition(oldPoint, URI, line, column, &show);
+
+ std::string explanation = "";
+
+ if (newMode != oldMode) {
+ explanation = "It was previously defined as a "s + DefinitionName[oldMode] + ". ";
+ } else if (newMode == Form::Function) {
+ explanation = "Please use #[Extend] attribute to add polymorphic forms, or #[Override] to replace prior definition.";
+ }
+
+ return ParserError(newPoint, "Can not redefine '" + qn +
+ "'; " + explanation + " Previous definition at "
+ + URI + "(" + std::to_string(line) + ":"
+ + std::to_string(column) + ")\n" + show
+ );
+ }
+
+ PartialDefinition RepositoryNode::Resolve(const std::string& qualifiedName, std::unordered_set<RepositoryNode*> &visited) {
+ static const PartialDefinition empty;
+ if (visited.count(this)) return empty;
+ visited.emplace(this);
+
+ PartialDefinition pd;
+
+ for (auto i : imports) {
+ pd.Append(i->Resolve(qualifiedName, visited));
+ }
+
+ auto c = changes.find(qualifiedName);
+ if (c != changes.end()) {
+ pd.Append(c->second);
+ }
+
+ return pd;
+ }
+
+ static std::unordered_set<std::string> CacheKeysToSet(const std::unordered_map<std::string, PartialDefinition>& pd) {
+ std::unordered_set<std::string> keys;
+ for (auto& c : pd) {
+ keys.emplace(c.first);
+ }
+ return keys;
+ }
+
+ void RepositoryNode::AddImport(RepositoryNode* i) {
+ if (std::find(imports.begin(), imports.end(), i) != imports.end()) {
+ return;
+ }
+ imports.emplace_back(i);
+ }
+
+ void RepositoryNode::Reset() {
+ imports.clear();
+ changes.clear();
+ }
+
+ void PartialDefinition::Append(const PartialDefinition& b) {
+ forms.insert(forms.end(), b.forms.begin(), b.forms.end());
+ recurData.insert(recurData.end(), b.recurData.begin(), b.recurData.end());
+ }
+
+ PartialDefinition PartialDefinition::Value(CGRef v) {
+ PartialDefinition pd;
+ pd.forms.emplace_back(Form::Val(v));
+ return pd;
+ }
+
+ Err<CGRef> PartialDefinition::Complete(Repository* r, const std::string& name) const {
+ if (forms.empty()) {
+ return (CGRef)nullptr;
+ }
+
+ Form::Mode mode = forms[0].mode;
+ Type fn{ forms[0].body };
+ Type rd{ false };
+
+ // validate extension and override
+ for (int i = 1; i < forms.size(); ++i) {
+
+ if (forms[i].HasAttr(Attributes::AlwaysOverride)) {
+ fn = Type{ false };
+ } else if (mode != forms[i].mode ||
+ !forms[i].HasAttr(Attributes::Extend)) {
+ if (forms[i].HasAttr(Attributes::MayOverride)) {
+ fn = Type{ false };
+ } else {
+ if (r) {
+ return r->RedefinitionError(
+ name,
+ forms[i].body->GetRepositoryAddress(),
+ forms[i].mode,
+ forms[i - 1].body->GetRepositoryAddress(),
+ forms[i - 1].mode
+ );
+ } else {
+ return ParserError(forms[i].body->GetRepositoryAddress(),
+ "Redefinition of '" + name + "'");
+ }
+ }
+ }
+ mode = forms[i].mode;
+ fn = Type::Pair(Type(forms[i].body), fn);
+
+ }
+
+ for (auto& r : recurData) {
+ rd = Type::Pair(r, rd);
+ }
+
+ switch (mode) {
+ case Form::Undefined:
+ return nullptr;
+ case Form::Macro:
+ return forms.back().body;
+ case Form::Function:
+ assert(forms.size());
+ break;
+ }
+
+ auto form = Type::User(
+ &FunctionTag,
+ Type::Tuple(
+ Type(name.c_str()),
+ rd, fn, Type{ false })
+ );
+
+ return Nodes::Invariant::Constant::New(form);
+ }
+
+ Err<symbol_t> PartialDefinition::CompleteMeta(Repository* r, const std::string& name) const {
+ LET_ERR(graph, Complete(r, name));
+ symbol_t sym;
+ sym.graph = graph;
+ for (auto& f : forms) {
+ for (auto& md : f.metadata) {
+ auto& aggr{ sym.metadata[md.first] };
+ if (aggr.size()) aggr += "\n";
+ aggr += md.second;
+ }
+ }
+ return sym;
+ }
+
+
+ Err<symbol_t> Repository::Build(const std::string& qualifiedName) {
+ RegionAllocator r{ stubs };
+ std::unordered_set<RepositoryNode*> vs;
+ return
+ root.Resolve(qualifiedName, vs).CompleteMeta(this, qualifiedName);
+ }
+
+ const symbol_t* Repository::Lookup(const std::string& qualifiedName) {
+ auto f = completeDefinitions.find(qualifiedName);
+ if (f != completeDefinitions.end()) return &f->second;
+ else return nullptr;
+ }
+
+ void Repository::InvalidateSymbolsInNode(RepositoryNode* n) {
+ for (auto s : n->changes) {
+ changed_symbols.emplace(s.first);
+ }
+ }
+
+ Err<void> Repository::UpdateDefinitions() {
+ if (changed_symbols.size()) {
+ for (auto& sym : changed_symbols) {
+ LET_ERR(complete, Build(sym));
+ completeDefinitions[sym] = complete;
+ }
+ }
+ return { };
+ }
+
+ static std::string ExtractVersionNumber(std::string& semver) {
+ auto dotPos = semver.find_first_of('.');
+ if (dotPos != semver.npos) {
+ semver = semver.substr(dotPos + 1);
+ }
+ return semver.substr(0, dotPos);
+ }
+
+ Err<void> Repository::CreateImportTask(RepositoryNode* parent, BufferKey uri, immediate_handler_t imm) {
+ if (uri.package == "#CORE") {
+ uri.package = defaultRepo;
+ uri.version = defaultVersion;
+ }
+
+ auto mod = modules.find(uri.package);
+ if (mod != modules.end()) {
+ auto importVersion = uri.version;
+ auto loadedVersion = mod->second.version;
+
+ std::string mostRecentVersion;
+
+ if (importVersion != loadedVersion) {
+ auto mi = ExtractVersionNumber(importVersion);
+ auto ml = ExtractVersionNumber(loadedVersion);
+
+ if (mi != ml) {
+ return ParserError(nullptr,
+ "Trying to import " + uri.version + " while " +
+ mod->second.version + " is already imported. The major "
+ "versions of these packages are incompatible.");
+ }
+
+ while(mostRecentVersion.empty()) {
+ auto iDigit = ExtractVersionNumber(importVersion);
+ auto mDigit = ExtractVersionNumber(loadedVersion);
+ if (iDigit.empty()) {
+ mostRecentVersion = mod->second.version;
+ } else if (mDigit.empty()) {
+ mostRecentVersion = uri.version;
+ } else if (strtoul(iDigit.c_str(),nullptr,10) >
+ strtoul(mDigit.c_str(), nullptr, 10)) {
+ mostRecentVersion = uri.version;
+ } else if (strtoul(iDigit.c_str(), nullptr, 10) <
+ strtoul(mDigit.c_str(), nullptr, 10)) {
+ mostRecentVersion = mod->second.version;
+ } else if (iDigit > mDigit) {
+ mostRecentVersion = uri.version;
+ } else if (iDigit < mDigit) {
+ mostRecentVersion = mod->second.version;
+ }
+ }
+
+ if (mostRecentVersion != uri.version) {
+ std::clog << "* Substituting version " << mostRecentVersion << " for " << uri.str() << " requested by the import.\n";
+ uri.version = mostRecentVersion;
+ }
+
+ if (mostRecentVersion != mod->second.version) {
+ std::clog << "* Upgrading [" << mod->first << " " << mod->second.version << "] to version " << mostRecentVersion << "\n";
+ for (auto& file : mod->second.files) {
+ importQueue.emplace_back(
+ ImportTask{
+ { mod->first, mostRecentVersion, file.first },
+ &file.second,
+ {},
+ relativeBasePath
+ }
+ );
+ }
+ mod->second.version = mostRecentVersion;
+ }
+ }
+
+ auto loadedFile = mod->second.files.find(uri.path);
+ if (loadedFile != mod->second.files.end()) {
+ // already loaded
+ parent->AddImport(&loadedFile->second);
+ InvalidateSymbolsInNode(&loadedFile->second);
+ return { };
+ }
+ } else {
+ mod = modules.emplace(
+ uri.package,
+ RepositoryModule{
+ {},
+ uri.version
+ }).first;
+ }
+
+ auto& newNode{ mod->second.files[uri.path] };
+
+ parent->AddImport(&newNode);
+
+ auto& importToNode{ mod->second.files[uri.path] };
+
+ importQueue.emplace_back(
+ ImportTask{
+ std::move(uri),
+ &importToNode,
+ imm,
+ relativeBasePath
+ }
+ );
+ return { };
+ }
+
+ static lithe::rule KronosParser = lithe::grammar::kronos::parser();
+
+ Err<void> Repository::ParseIntoNode(RepositoryNode* node, const std::string& tmpCode, immediate_handler_t imm) {
+ // store state for rollback
+ rollback.emplace(node, *node);
+
+ InvalidateSymbolsInNode(node);
+ node->Reset();
+ node->retainSource = tmpCode;
+
+ auto& code{ node->retainSource };
+
+ const char* sb = code.data();
+ const char* se = code.data() + code.size();
+ auto parseTree = (*KronosParser)(sb, se);
+
+ if (parseTree.is_error()) {
+ std::stringstream msg;
+ parseTree.to_stream(msg);
+ return ParserError(parseTree.children.size()
+ ? parseTree.children.front().strbeg
+ : nullptr,
+ msg.str());
+ }
+
+ RegionAllocator allocateFrom{ node->graphs };
+
+ LET_ERR(syms, (GenerateSymbols(
+ parseTree,
+ node->incremental,
+ [this, node](BufferKey imp) -> Err<void> {
+ return CreateImportTask(node, imp, { });
+ },
+ [imm](const char* sym, CGRef expr) -> Err<void> {
+ if (imm) imm(sym, expr);
+ return { };
+ })));
+
+ for (auto& def : syms) {
+ changed_symbols.emplace(def.first);
+ node->changes[def.first].Append(def.second);
+
+ if (node->canOverwrite) {
+ for (auto& f : node->changes[def.first].forms) {
+ f.attr = (Attributes)((int)Attributes::MayOverride | (int)f.attr);
+ }
+ }
+ }
+ return { };
+ }
+
+ struct RestoreString {
+ std::string o;
+ std::string& n;
+ RestoreString(std::string& s) :n(s) {
+ o = n;
+ }
+ ~RestoreString() {
+ n = o;
+ }
+ };
+
+ Err<void> Repository::Perform(ImportTask it) {
+ RestoreString folderScope{ relativeBasePath };
+ relativeBasePath = it.contextPath;
+
+ std::string pathToFile;
+
+ if (it.uri.package == "#LOCAL") {
+ if (std::regex_match(it.uri.path, std::regex("^([a-zA-Z+]:)?[/\\\\].*"))) {
+ pathToFile = it.uri.path;
+ } else {
+ pathToFile = relativeBasePath + it.uri.path;
+ }
+ } else {
+ pathToFile = TLS::GetCurrentInstance()->ResolveModulePath(
+ it.uri.package.c_str(),
+ it.uri.path.c_str(),
+ it.uri.version.c_str());
+ }
+
+ std::ifstream readFile{ pathToFile };
+ if (readFile.is_open()) {
+
+ relativeBasePath = GetCanonicalAbsolutePath(GetParentPath(pathToFile));
+ it.node->fileSystemPath = GetCanonicalAbsolutePath(pathToFile);
+
+ std::stringstream rs;
+ rs << readFile.rdbuf();
+
+ InvalidateSymbolsInNode(it.node);
+ it.node->Reset();
+
+ it.node->URI = it.uri;
+ auto check = ParseIntoNode(it.node, rs.str(), it.imm);
+ if (check.err) {
+ std::string importChain;
+
+ importChain = " while importing " + it.node->URI.str();
+
+ return ParserError(
+ check.err->GetSourceFilePosition(),
+ check.err->GetErrorMessage() + importChain
+ );
+ } else {
+ return check;
+ }
+ } else {
+ return FileError("Could not open " + it.uri.str() + " (" + pathToFile + ")");
+ }
+ return { };
+ }
+
+ static void DumpDot(std::ostream& dot, RepositoryNode* node, std::unordered_set<RepositoryNode*>& visited) {
+ bool first = visited.empty();
+ if (visited.count(node)) return;
+ visited.emplace(node);
+
+ if (first) {
+ dot << "\n\ndigraph dependencies {\n\tnode [shape=box]\n";
+ }
+
+ dot << "\tn" << (uintptr_t)node << " [label=\"" << node->URI.str() << "\"]\n";
+
+ for (auto i : node->imports) {
+ dot << "\tn" << (uintptr_t)i << " -> n" << (uintptr_t)node << "\n";
+ DumpDot(dot, i, visited);
+ }
+
+ if (first) {
+ dot << "}\n\n";
+ }
+ }
+
+ Err<void> Repository::Transaction(std::function<Err<void>()> tx) {
+ assert(rollback.empty());
+ changed_symbols.clear();
+ Err<void> success;
+
+ success = tx();
+ if (success.err) {
+
+ importQueue.clear();
+
+ for (auto& rb : rollback) {
+ *rb.first = rb.second;
+ }
+
+ for (auto& mod : modules) {
+ for (auto i = mod.second.files.begin(); i != mod.second.files.end();) {
+ auto thisFile = i++;
+ if (thisFile->second.changes.empty()) {
+ mod.second.files.erase(thisFile);
+ }
+ }
+ }
+
+ changed_symbols.clear();
+ }
+ rollback.clear();
+ return success;
+ }
+
+ Err<void> Repository::PerformQueue() {
+ while (importQueue.size()) {
+ auto it = importQueue.back();
+ importQueue.pop_back();
+ CHECK_ERR(Perform(it));
+ }
+#if 0
+ std::unordered_set<RepositoryNode*> vs;
+ DumpDot(std::cout, &root, vs);
+#endif
+
+ return UpdateDefinitions();
+ }
+
+ Err<void> Repository::ImportTree(BufferKey fromLeaf, immediate_handler_t imm) {
+ relativeBasePath = "";
+ assert(importQueue.empty());
+
+ CHECK_ERR(CreateImportTask(&root, fromLeaf, imm));
+ return PerformQueue();
+ }
+
+ Repository::Repository() {
+ const char* repo = getenv("KRONOS_CORE_REPOSITORY");
+ const char* repoVersion = getenv("KRONOS_CORE_REPOSITORY_VERSION");
+ if (!repo) repo = KRONOS_CORE_LIBRARY_REPOSITORY;
+ if (!repoVersion) repoVersion = KRONOS_CORE_LIBRARY_VERSION;
+
+ defaultRepo = repo;
+ defaultVersion = repoVersion;
+
+ root.AddImport(&kernel);
+ root.URI = { "< root", "context", ">" };
+ kernel.URI = { repo, "#CORE", repoVersion };
+ }
+
+ Err<void> Repository::ImportFile(const char* path, bool canOverwrite) {
+ return Transaction([=](){ return ImportTree({ "#LOCAL", "", path }, { }); });
+ }
+
+ Err<void> Repository::ImportBuffer(const char* sourceCode, bool canOverwrite, immediate_handler_t imm) {
+ return Transaction([=]() -> Err<void> {
+ relativeBasePath = "";
+ auto& newNode = *adhoc.emplace(adhoc.end());
+ newNode.canOverwrite = true;
+ CHECK_ERR(ParseIntoNode(&newNode, sourceCode, imm));
+ root.AddImport(&newNode);
+ InvalidateSymbolsInNode(&newNode);
+ return PerformQueue();
+ });
+ }
+
+ Err<void> Repository::ImportCoreLib(const char* file) {
+ return Transaction([=]() {
+ return ImportTree({ "#CORE", "", file }, { });
+ });
+ }
+
+ void Repository::Rebind(const std::string& qn, Nodes::CGRef expr) {
+ completeDefinitions[qn].graph = expr;
+ }
+
+ RepositoryBuilder Repository::GetKernelBuilder() {
+ return RepositoryBuilder{ ":", *this };
+ }
+
+ RepositoryBuilder::~RepositoryBuilder() {
+ }
+
+ void Repository::GetPosition(const char* memPos, std::string& uri, int& line, int& column, std::string* show_line) {
+ for (auto& m : modules) {
+ for (auto& f : m.second.files) {
+ auto& src{ f.second.retainSource };
+ if (src.data() < memPos &&
+ src.data() + src.size() > memPos) {
+
+ uri = f.second.fileSystemPath;
+ line = 0;
+ column = 0;
+
+ const char* lineBeg = src.data();
+ const char* pos = src.data();
+ while(pos < memPos) {
+ if (*pos++ == '\n') {
+ ++line; column = 0;
+ lineBeg = pos;
+ } else {
+ ++column;
+ }
+ }
+
+ if (show_line) {
+ while (*pos && *pos != '\n') ++pos;
+ while (isspace(*lineBeg) && lineBeg < pos) {
+ ++lineBeg; --column;
+ }
+ *show_line = "\t" + std::string(lineBeg, pos) + "\n\t"
+ + std::string(column, ' ') + "^^^\n";
+ }
+ return;
+ }
+ }
+ }
+ uri.clear(); line = column = -1;
+ }
+
+ void RepositoryBuilder::AddMacro(const char* relative, CGRef val, bool thing) {
+ std::string sym = path + relative;
+ r.kernel.changes[sym].forms.emplace_back(Form::Val(val));
+ r.changed_symbols.emplace(sym);
+ }
+
+ void RepositoryBuilder::AddFunction(const char* relative, CGRef val, const char* args, const char* doc, const char* fallback) {
+ std::string sym = path + relative;
+ if (fallback) {
+ AddFunction(
+ relative,
+ Evaluate::New(fallback,
+ Lib::Reference::New({ fallback }),
+ GenericPair::New(Lib::Reference::New({ sym }),
+ GenericPair::New(Invariant::Constant::New(Type(relative)),
+ GenericArgument::New()))));
+ }
+
+ Form f = Form::Fn(val);
+ f.attr = Attributes::Extend;
+ if (args && doc && strlen(doc)) {
+ f.metadata[args] = doc;
+ }
+ r.kernel.changes[sym].forms.emplace_back(f);
+ r.changed_symbols.emplace(sym);
+ }
+
+ MemoryRegion* RepositoryBuilder::GetMemoryRegion() {
+ return r.kernel.graphs;
+ }
+ }
+}
+
+/*
+
+ Import buffer
+
+ - construct local definitions -> changes
+ - push invalidates to importees (changes)
+ - walk imports (will push invalidates to me)
+
+ - after import cycle, root should have all the invalidates. Then validate the constructs.
+*/
No newline at end of file
M src/k3/FlowControl.cpp +0 -1
@@ 421,7 421,6 @@ namespace K3 {
void BuildSelectPrimitiveOps(Parser::RepositoryBuilder pack) {
using namespace Nodes;
/* Select primitives */
- RegionAllocator KernelNodes;
CGRef Arg(GenericArgument::New());
CGRef vec(GenericFirst::New(Arg)),idx(Convert::New(Convert::Int32,GenericRest::New(Arg)));
CGRef lenc(GenericGetVectorLen::New(vec));
M src/k3/Generic.cpp +11 -7
@@ 123,13 123,17 @@ namespace K3 {
template <typename STRI> void EscapeStringXML(std::ostream& strm, const STRI& beg, const STRI& end) {
for (auto i(beg); i != end; ++i) {
- switch (*i) {
- case '\"':strm << """; break;
- case '<':strm << "<"; break;
- case '>':strm << ">"; break;
- case '&':strm << "&"; break;
- case '\'': strm << "'"; break;
- default:strm << (char)*i; break;
+ if (iscntrl(*i)) {
+ strm << "&#" << (int)* i << ";";
+ } else {
+ switch (*i) {
+ case '\"':strm << """; break;
+ case '<':strm << "<"; break;
+ case '>':strm << ">"; break;
+ case '&':strm << "&"; break;
+ case '\'': strm << "'"; break;
+ default:strm << (char)* i; break;
+ }
}
}
}
M src/k3/Invariant.cpp +9 -10
@@ 426,7 426,7 @@ namespace K3 {
template <class RETURN, class ARG1, class ARG2>
- void AddBinary(Parser::RepositoryBuilder pack, const char *symbol, RETURN(*func)(ARG1,ARG2),const char *arglist = 0, const char *comment = 0)
+ void AddBinary(Parser::RepositoryBuilder& pack, const char *symbol, RETURN(*func)(ARG1,ARG2),const char *arglist = 0, const char *comment = 0)
{
using namespace Invariant;
if (arglist == 0) arglist = "a b";
@@ 449,14 449,14 @@ namespace K3 {
(func,inv_l,inv_r,symbol,a,b);
}
- void AddBinary(Parser::RepositoryBuilder pack, const char *symbol, CGRef graph, const char *arglist, const char *comment)
+ void AddBinary(Parser::RepositoryBuilder& pack, const char *symbol, CGRef graph, const char *arglist, const char *comment)
{
if (arglist == 0) arglist = "a b";
pack.AddFunction(symbol,graph,arglist,comment);
}
template <class RETURN, class ARG1, class ARG2>
- void AddBinaryInversible(Parser::RepositoryBuilder pack, const char *symbol,
+ void AddBinaryInversible(Parser::RepositoryBuilder& pack, const char *symbol,
RETURN(*func)(ARG1,ARG2),
ARG1(*inv_l)(RETURN,ARG2),
ARG2(*inv_r)(RETURN,ARG1),
@@ 472,7 472,7 @@ namespace K3 {
template <class RETURN,class ARG>
- void AddUnary(Parser::RepositoryBuilder pack, const char *symbol, RETURN(*func)(ARG), const char *arglist = 0, const char *comment = 0)
+ void AddUnary(Parser::RepositoryBuilder& pack, const char *symbol, RETURN(*func)(ARG), const char *arglist = 0, const char *comment = 0)
{
using namespace Invariant;
if (arglist == 0) arglist = "a";
@@ 813,7 813,7 @@ namespace K3 {
};
};
- void BuildInvariantLogic(Parser::RepositoryBuilder pack)
+ void BuildInvariantLogic(Parser::RepositoryBuilder& pack)
{
using namespace K3::Nodes;
using namespace K3::Nodes::Invariant;
@@ 926,7 926,7 @@ namespace K3 {
};
};
- void BuildInvariantArithmetic(Parser::RepositoryBuilder pack)
+ void BuildInvariantArithmetic(Parser::RepositoryBuilder& pack)
{
using namespace K3::Nodes;
using namespace K3::Nodes::Invariant;
@@ 954,9 954,8 @@ namespace K3 {
}
- void BuildInvariantPrimitiveOps(Parser::RepositoryBuilder pack)
+ void BuildInvariantPrimitiveOps(Parser::RepositoryBuilder& pack)
{
- RegionAllocator KernelNodes;
BuildInvariantArithmetic(pack);
BuildInvariantLogic(pack);
@@ 1007,8 1006,8 @@ namespace K3 {
pack.AddMacro("Constant",GenericTypeTag::New(&InvariantTag), false);
BuildInvariantStringOps(pack.AddPackage("String"));
- AddBinary(pack, "Get-Library-Metadata", GenericGetLibraryMetadata::New(Arg), "package", "");
- AddBinary(pack, "Get-Symbol-Source", GenericGetSymbolSource::New(Arg), "symbol", "");
+// AddBinary(pack, "Get-Library-Metadata", GenericGetLibraryMetadata::New(Arg), "package", "");
+// AddBinary(pack, "Get-Symbol-Source", GenericGetSymbolSource::New(Arg), "symbol", "");
AddBinary(pack, "Specialization-Trace", Trace::New(Arg), "fn", "");
pack.AddMacro("Reject-All-Forms", GenericNoFallback::New(), false);
M src/k3/Invariant.h +1 -1
@@ 172,5 172,5 @@ namespace K3 {
};
};
- void BuildInvariantPrimitiveOps(Parser::RepositoryBuilder pack);
+ void BuildInvariantPrimitiveOps(Parser::RepositoryBuilder& pack);
};
No newline at end of file
M src/k3/Native.cpp +0 -2
@@ 799,7 799,6 @@ namespace K3 {
void BuildNativeMathFuncs(Parser::RepositoryBuilder pack) {
using namespace Nodes;
using namespace Nodes::Native;
- RegionAllocator PrimitiveOpAllocator;
auto arg = GenericArgument::New();
auto b1 = GenericFirst::New(arg);
auto b2 = GenericRest::New(arg);
@@ 821,7 820,6 @@ namespace K3 {
void BuildNativePrimitiveOps(Parser::RepositoryBuilder pack) {
using namespace Nodes;
using namespace Nodes::Native;
- RegionAllocator PrimitiveOpAllocator;
auto arg = GenericArgument::New();
auto b1 = GenericFirst::New(arg);
auto b2 = GenericRest::New(arg);
M src/k3/NativeVector.cpp +0 -1
@@ 80,7 80,6 @@ namespace K3 {
void BuildVectorPrimitiveOps(Parser::RepositoryBuilder pack) {
using namespace Nodes;
- RegionAllocator VectorOpAllocator;
auto arg = GenericArgument::New();
auto b1 = GenericFirst::New(arg);
auto b2 = GenericRest::New(arg);
M src/k3/Parser.cpp +176 -425
@@ 56,6 56,11 @@ namespace K3 {
resolver_map_t& scope_symbols;
std::list<std::pair<std::string, std::vector<Lib::Reference*>>> captured_syms;
capture_link* previous;
+
+ ~capture_link() {
+
+ }
+
bool is_free(const std::string& sym) const {
return scope_symbols.count(sym) == 0;
}
@@ 112,16 117,36 @@ namespace K3 {
return ast_node->GetRepositoryAddress();
}
+ const char* get_position(const PartialDefinition& pd) {
+ for (auto& f : pd.forms) {
+ auto pos = get_position(f.body);
+ if (pos) return pos;
+ }
+ return nullptr;
+ }
+
CGRef fn_apply(const char* name, const char *name_end, CGRef fn, CGRef args) {
return Evaluate::New(name, fn, args, name_end);
}
- Err<CGRef> generate_fn(std::string ns, std::string name, CI beg, CI end, const AstNode& args, const AstNode& attributes, capture_link* cl, parser_state_t lookup);
+ Err<PartialDefinition> generate_fn(std::string ns, std::string name, CI beg, CI end, const AstNode& args, const AstNode& attributes, capture_link* cl, parser_state_t lookup);
#define PROPAGATE_ERR(statement) { auto err = statement; if (err.err) return std::move(err.err); }
+ static void defineFn(resolver_map_t& defs, std::string sym, const AstNode& c, std::function<Err<PartialDefinition>()> f) {
+ resolution_t def{&c, std::move(f)};
+ defs[sym].emplace_back(std::move(def));
+ }
+
static void define(resolver_map_t& defs, std::string sym, const AstNode& c, std::function<Err<CGRef>()> f) {
- defs[sym].emplace_back(resolution_t{ c, std::move(f) });
+ resolution_t def{
+ &c,
+ [f]() -> Err<PartialDefinition> {
+ LET_ERR(val, f());
+ return PartialDefinition::Value(val);
+ }
+ };
+ defs[sym].emplace_back(std::move(def));
}
static Err<void> destructure(resolver_map_t& defs, const AstNode& c, std::function<Err<CGRef>()> rhs) {
@@ 183,7 208,7 @@ namespace K3 {
return Raise::New(Invariant::Constant::New(Type::Nil));
}
auto err = n.back();
- for (int i = n.size() - 2; i >= 0; --i) {
+ for (int i = (int)n.size() - 2; i >= 0; --i) {
err = GenericPair::New(n[i], err);
}
return Raise::New(err);
@@ 239,20 264,22 @@ namespace K3 {
return "Infix" + op;
}
- Err<CGRef> cons_fn(const std::string& ns, const std::string& name, const AstNode& args, CI beg, CI end, AstNode attributes = {}) {
+ Err<PartialDefinition> cons_fn(const std::string& ns, const std::string& name, const AstNode& args, CI beg, CI end, AstNode attributes = {}) {
auto result = generate_fn(ns, name, beg, end, args, attributes, cl, pst);
if (result.err) {
return ParserError(result.err->GetSourceFilePosition(),
result.err->GetErrorMessage() + ", while parsing "s + ns + name);
}
- return result;
+ return *result;
};
#define CASE(ty) if (n.strbeg == tag :: ty)
Err<CGRef> cons(const AstNode& n) {
auto binding(CurrentSourcePosition = get_position(n));
CASE(quote) {
- return cons_fn("", "", AstNode(), n.children.begin(), n.children.end());
+ LET_ERR(afn, cons_fn("", "", AstNode(), n.children.begin(), n.children.end()));
+ LET_ERR(fn, afn.Complete(nullptr));
+ return fn;
}
CASE(leftarrow) {
AstNode body(tag::body);
@@ 261,14 288,19 @@ namespace K3 {
LET_ERR(a, cons(n[1]))
LET_ERR(sym, refer(":Invoke-With", get_position(n)))
- LET_ERR(fn, cons_fn("", "", n[0], body.children.begin(), body.children.end()))
+ LET_ERR(afn, cons_fn("", "", n[0], body.children.begin(), body.children.end()))
+ LET_ERR(fn, afn.Complete(nullptr));
return fn_apply("<-",nullptr, sym,
GenericPair::New(a, fn));
}
CASE(infix) {
std::string sym, op = read_string(n[1]);
- if (op == "=>") return cons_fn("", "", n[0], n.children.begin() + 2, n.children.end());
+ if (op == "=>") {
+ LET_ERR(afn, cons_fn("", "", n[0], n.children.begin() + 2, n.children.end()));
+ LET_ERR(fn, afn.Complete(nullptr));
+ return fn;
+ }
LET_ERR(lhs, cons(n[0]));
LET_ERR(rhs, cons(n[2]));
sym = get_infix_fn(op);
@@ 350,7 382,8 @@ namespace K3 {
if (n.children.empty()) {
return ParserError(get_position(n), "Function contains no forms");
}
- return cons_fn("", "", AstNode(), n.children.begin(), n.children.end());
+ LET_ERR(afn, cons_fn("", "", AstNode(), n.children.begin(), n.children.end()));
+ return afn.Complete(nullptr);
}
CASE(sec_right) {
AstNode tmp(tag::infix);
@@ 452,8 485,7 @@ namespace K3 {
CASE(type) {
std::string name(read_string(n[0]));
std::string type_name = pstate.namespaces[0] + name;
- const char *pos = get_position(n);
- define(local_syms, name, n, [type_name,pos]() {
+ define(local_syms, name, n, [type_name,pos = get_position(n)]() {
auto binding(CurrentSourcePosition = pos);
return GenericTypeTag::New(type_name);
});
@@ 469,7 501,7 @@ namespace K3 {
auto slash = name.find_first_of('/');
auto subname = name.substr(0, slash);
auto ns = scope.previous ? "" : pstate.namespaces.front();
- define(local_syms, subname, n, [ns,expr_cons,subname,&n]() mutable -> Err<CGRef> {
+ defineFn(local_syms, subname, n, [ns,expr_cons,subname,&n]() mutable -> Err<PartialDefinition> {
auto binding(CurrentSourcePosition = get_position(n));
if (n[1].children.empty()) {
return ParserError(CurrentSourcePosition, "Empty function!");
@@ 503,7 535,9 @@ namespace K3 {
CASE(docstring) {
continue;
}
+
ext = hooks.find(tag::immediate);
+
if (ext != hooks.end()) {
auto err = ext->second(n);
if (err.err) return err;
@@ 531,7 565,8 @@ namespace K3 {
for (auto &ss : lexical_scope.scope_symbols) {
for (auto &d : ss.second) {
LET_ERR(value, d.second());
- bindings[ss.first].emplace_back(value);
+ LET_ERR(expr, value.Complete(nullptr));
+ bindings[ss.first].emplace_back(expr);
}
}
@@ 604,7 639,7 @@ namespace K3 {
return false;
};
- Err<CGRef> generate_fn(std::string sym_ns, std::string sym, CI beg, CI end, const AstNode& args, const AstNode& attributes, capture_link* cl, parser_state_t pst) {
+ Err<PartialDefinition> generate_fn(std::string sym_ns, std::string sym, CI beg, CI end, const AstNode& args, const AstNode& attributes, capture_link* cl, parser_state_t pst) {
resolver_map_t defs;
capture_link scope{ defs, {}, cl };
static const string ret_val = "return value"; // due to whitespace this can't collide with program defs
@@ 644,7 679,6 @@ namespace K3 {
scope.scope_symbols.erase(sym);
}
- //std::cout << "\n[" << sym << "]\n";
LET_ERR(bindings, resolve_local_bindings(scope));
for (auto &b : bindings) {
@@ 669,22 703,24 @@ namespace K3 {
arg_root->Resolve(GenericArgument::New());
}
- Type graph_list;
- Type recur_pts;
+ PartialDefinition pd;
if (!has_attribute(attributes, "Extend")) {
- graph_list = Type(Evaluate::CallLib(":Fallback:Specialization-Type-Mismatch", arg_root));
+ pd.forms.emplace_back(Form::Fn(
+ Evaluate::CallLib(":Fallback:Specialization-Type-Mismatch", arg_root)));
}
if (has_attribute(attributes, "Pattern")) {
- if (graph_list.IsNil()) {
+ if (pd.forms.empty()) {
return ParserError(get_position(attributes),
"Extend and Pattern attributes are mutually exclusive");
}
- graph_list = Type(Invariant::GenericPropagateFailure::New());
+ pd.forms.clear();
+ pd.forms.emplace_back(Form::Fn(
+ Invariant::GenericPropagateFailure::New()));
+ pd.forms.back().attr = Attributes::Pattern;
}
-
for (auto& binding : bindings[ret_val]) {
// dedupe graph
ShareSubgraphs dedupe{ binding, recur };
@@ 693,30 729,44 @@ namespace K3 {
for (auto ev : Qxx::FromGraph(formGraph).OfType<Nodes::Evaluate>()) {
Lib::Reference* ref;
if (ev->GetUp(0)->Cast(ref) && (ref == recur || ref->GetName() == "Recur")) {
- recur_pts = Type::Pair(Type(ev), recur_pts);
+ pd.recurData.emplace_back(Type(ev));
}
}
- Type form = Type(formGraph);
- graph_list = graph_list.IsNil() ? form : Type::Pair(form, graph_list);
+ Form f;
+ f.mode = Form::Function;
+ f.attr = Attributes::Extend;
+ f.body = formGraph;
+ pd.forms.emplace_back(f);
}
- CGRef fn = Invariant::Constant::New(
- pack_fn(sym.c_str(), graph_list, recur_pts));
-
+ if (pd.forms.size() && has_attribute(attributes, "Override")) {
+ if (has_attribute(attributes, "Extend")) {
+ return ParserError(get_position(attributes),
+ "Extend and Override attributes are mutually exclusive");
+ }
+ pd.forms[0].attr = Attributes::AlwaysOverride;
+ }
+
+
+ LET_ERR(fn, pd.Complete(nullptr, sym));
if (scope.captured_syms.size()) {
CGRef curry_args = Invariant::Constant::New(false);
for (auto csi = scope.captured_syms.rbegin();csi != scope.captured_syms.rend();++csi) {
curry_args = GenericPair::New(Lib::Reference::New({ csi->first }), curry_args);
}
fn = fn_apply(":Curry", nullptr, Lib::Reference::New({ ":Curry" }), GenericPair::New(fn, curry_args));
+ // closures need to be defined as values rather than multimorphs because
+ // of the enclosing curry
+ pd = PartialDefinition::Value(fn);
}
if (sym_ns.empty()) {
fn->HasInvisibleConnections();
recur->Resolve(fn);
}
- return fn;
+
+ return pd;
}
static void write_args(std::ostream& s, const AstNode& n) {
@@ 735,291 785,73 @@ namespace K3 {
if (n.strbeg == tag::list) s << "]";
}
- static Type extend_fn(const Type& new_forms, const Type& old_forms) {
- if (old_forms.IsNil()) return new_forms;
- if (new_forms.IsNil()) return old_forms;
- assert(new_forms.IsUserType(FunctionTag));
- Type aname, arecur, aforms, bname, brecur, bforms;
- new_forms.UnwrapUserType().Tie(aname, arecur, aforms);
- if (old_forms.IsUserType(FunctionTag)) {
- old_forms.UnwrapUserType().Tie(bname, brecur, bforms);
- } else bforms = old_forms;
- return Type::User(&FunctionTag, Type::Tuple(aname,
- Type::Append(arecur, brecur),
- Type::Append(aforms, bforms), Type::Nil));
- }
-
- const symbol_t* Repository::lookup(const std::string& qualified_name) {
- auto sym = symbol_cache.find(qualified_name);
- if (sym == symbol_cache.end()) {
- if (symbols.count(qualified_name) == 0) return nullptr;
- sym = symbol_cache.emplace(qualified_name, build_symbol(qualified_name)).first;
+ static Err<void> Reify(std::string ns, DefinitionMap& resolved, const resolver_map_t& defs) {
+ for (auto& ds : defs) {
+ PartialDefinition pd;
+ for (auto& d : ds.second) {
+ LET_ERR(resolution, d.second());
+ pd.Append(resolution);
+ }
+ resolved.emplace(ns + ds.first, pd);
}
- return &sym->second;
- }
-
- void Repository::temporary_rebind(const std::string& qualified_name, Nodes::CGRef expr) {
- symbol_cache[qualified_name].graph = expr;
- }
-
- symbol_t Repository::build_symbol(const std::string& qualified_name) {
- symbol_t sym;
- if (symbols.count(qualified_name)) {
- Type fn;
- for (auto& change : symbols[qualified_name]) {
- if (change.second.is_defn) {
- for (auto &md : change.second.metadata) {
- sym.metadata[md.first] += "\n" + md.second;
- }
- Type new_fn;
- Invariant::Constant *cgraph;
- for (auto &expr : change.second.expr) {
- if (expr->Cast(cgraph)) {
- new_fn = cgraph->GetType();
- fn = extend_fn(new_fn, fn);
- } else {
- INTERNAL_ERROR("Bad function extension");
- }
- }
- } else {
- sym.metadata = change.second.metadata;
- sym.graph = change.second.expr.front();
- fn = Type::Nil;
- }
- }
-
- if (!fn.IsNil()) {
- sym.graph = Invariant::Constant::New(fn);
- }
- }
-
- return sym;
+ return { };
}
- Err<void> Repository::emplace_syms(changeset_ref changes, const std::string& ns, const resolver_map_t& defs, std::unordered_set<std::string>& touches) {
- assert(&changes->second.rep == this);
- for (auto &pf : defs) {
- std::string qn = pf.first.front() == ':' ? pf.first : ns + pf.first;
- for (auto &d : pf.second) {
- auto &change = symbols[qn][changes->second.sequence];
-
- change.overwrite = changes->second.allow_overwrite;
-
- if (d.first.strbeg == tag::defn) {
- bool is_legal = change.expr.empty() && symbols[qn].size() == 1;
+ Err<void> GeneratePackage(DefinitionMap& parentDefs, std::string name, const CI& beg, const CI& end, parser_state_t state) {
+ state.namespaces.push_front(name);
- if (d.first[0].children.size() > 2) {
- if (has_attribute(d.first[0][2], "Extend")) {
- if (is_legal) {
- return ParserError(Parser::get_position(d.first), "Can't extend '" + qn + "' with no prior definitions. Use #[Override] to ignore the previous definition, or #[Pattern] attribute to delegate specialization failures to the caller.");
- }
- is_legal = true;
- }
- if (has_attribute(d.first[0][2], "Override")) {
- is_legal = true;
- }
- }
-
- if (change.overwrite) is_legal = true;
-
- if (!is_legal) {
- const char *prev = nullptr;
- if (change.expr.size()) prev = change.expr.front()->GetRepositoryAddress();
- else if (symbols[qn].size()) {
- for (auto &c : symbols[qn]) {
- if (c.first != changes->second.sequence) {
- if (c.second.expr.size()) {
- prev = c.second.expr.front()->GetRepositoryAddress();
- break;
- }
- }
- }
- }
+ resolver_map_t defs;
+ capture_link scope{ defs, { }, nullptr };
- std::string prevDef = "";
- if (prev) {
- std::string uri, show;
- int line, column;
- get_position(prev, uri, line, column, &show);
- if (uri.size()) {
- prevDef = "\nPrevious definition at " + uri + "(" +
- std::to_string(line) + ":" + std::to_string(column) + ")\n" + show;
- }
- }
-
- return ParserError(
- Parser::get_position(d.first), "Multiple definitions for " + qn + "; use #[Extend] attribute to append forms." + prevDef);
- }
-
- change.is_defn = true;
+ CHECK_ERR((generate_syms(beg, end, scope, state, {
+ {
+ tag::package,
+ [&](const AstNode& n) {
+ auto packname = read_string(n[0]);
+ if (packname.front() != ':') packname = name + packname;
+ return GeneratePackage(parentDefs, packname + ":", ++n.children.begin(),n.children.end(), state);
+ }
+ }})));
- std::string doc;
- if (d.first[1].children.size()) {
- if (d.first[1].children.back().strbeg == tag::docstring) {
- for (auto& ds : d.first[1].children.back().children) {
- if (doc.size()) doc.push_back(' ');
- doc += read_string(ds);
- }
- }
- }
- std::stringstream args;
- write_args(args, d.first[0][1]);
- if (change.metadata[args.str()].size()) change.metadata[args.str()].push_back('\n');
- change.metadata[args.str()] += doc;
- } else {
- if (change.expr.size()) return ParserError(Parser::get_position(d.first), "Multiple definitions for " + qn);
- }
- LET_ERR(value, d.second());
- change.expr.emplace_back(value);
- touches.emplace(qn);
- symbol_cache.erase(qn);
- }
- }
- return {};
+ return Reify(name, parentDefs, defs);
}
- Err<void> Repository::generate_package(changeset_ref changes, std::string name, const CI& beg, const CI& end, parser_state_t state, bool allow_overwrite) {
- state.namespaces.push_front(name);
-
+
+ Err<DefinitionMap> GenerateSymbols(const AstNode& p,
+ parser_state_t& state,
+ std::function<Err<void>(BufferKey)> importHook,
+ immediate_handler_t imm) {
resolver_map_t defs;
capture_link scope{ defs,{},nullptr };
-
- PROPAGATE_ERR(generate_syms(beg, end, scope, state, {
- { tag::package,[&](const AstNode& n) {
- auto packname = read_string(n[0]);
- if (packname.front() != ':') packname = name + packname;
- return generate_package(changes, packname + ":", ++n.children.begin(),n.children.end(), state, allow_overwrite);
- } }
- }));
-
- return emplace_syms(changes, name, scope.scope_symbols, changed_symbols);
- }
-
- enum version_comparison {
- older = -1,
- unordered = 0,
- newer = 1,
- incompatible
- };
-
- static version_comparison compare_semver(const std::string& a, const std::string &b) {
- auto split = [](const std::string& a) {
- std::vector<int> nums;
- for (size_t pos = 0, prev = 0; a.npos != (pos = a.find('.', prev)); prev = pos + 1) {
- std::string item = a.substr(prev, pos - prev);
- nums.emplace_back(strtoul(item.c_str(), nullptr, 10));
- }
- return nums;
- };
- auto vera = split(a), verb = split(b);
-
- if (vera.size() && verb.size() && vera.front() != verb.front()) return incompatible;
- for (size_t i = 1;i < std::min(vera.size(), verb.size());++i) {
- if (vera[i] > verb[i]) return newer;
- if (vera[i] < verb[i]) return older;
- }
- return unordered;
- }
-
- void Repository::revert(changeset_ref cs) {
- for (auto &sym : symbols) {
- auto f = sym.second.find(cs->second.sequence);
- if (f != sym.second.end()) {
- changed_symbols.emplace(sym.first);
- symbol_cache.erase(sym.first);
- sym.second.erase(f);
- }
- }
- }
-
- Err<changeset_ref> Repository::new_changeset(const package_uri& uri, const std::string& file, bool allow_overwrite) {
- auto similar = changesets.equal_range(uri.uri);
- for(auto i = similar.first; i != similar.second;) {
- switch(compare_semver(uri.version, i->second.source.version)) {
- case older:
- return ParserError(nullptr, "* Not downgrading " + uri.identifier + " [" + uri.uri + " " +
- i->second.source.version + " -> " + uri.version + "]\n");
- break;
- case newer:
- std::clog << "* Upgrading " << uri.identifier << " [" << uri.uri << " "
- << i->second.source.version << " -> " << uri.version << "]\n";
- revert(i);
- changesets.erase(i++);
- break;
- case unordered:
- i++;
- break;
- case incompatible:
- return ParserError(nullptr, "* Major version clash [" + uri.uri + " " + uri.version + " <-> " + i->second.source.version + "]");
- break;
- }
- }
-
- return changesets.emplace(uri.uri, changeset_t{
- changeset_sequence++,
- uri,
- allow_overwrite,
- *this
- });
- }
-
- Err<void> Repository::import_buffer(const package_uri& uri, parser_state_t& state, const std::string& id, const std::string& text, immediate_handler_t imm, bool allow_overwrite) {
- LET_ERR(changes, new_changeset(uri, id, allow_overwrite))
-
- module_resources[id] = text;
- const char *sb = module_resources[id].data();
- const char *se = module_resources[id].data() + module_resources[id].size();
- module_lookup[module_resources[id].data()] = module_resources.find(id);
- auto p = (*parser)(sb, se);
-
- if (p.is_error()) {
- std::stringstream msg;
- p.to_stream(msg);
- revert(changes);
- ParserError err(p.children.size() ? p.children.back().strbeg : nullptr, msg.str());
- return std::move(err);
- }
-
- RegionAllocator parsed_asts;
-
- resolver_map_t defs;
- capture_link scope{ defs,{},nullptr };
+ std::unordered_map<std::string, PartialDefinition> resolved;
auto err = (generate_syms(p.children.begin(), p.children.end(), scope, state, {
{ tag::package,[&](const AstNode& n) -> Err<void> {
- auto packname = read_string(n[0]);
- if (packname.front() != ':') packname = ":" + packname;
- return generate_package(changes, packname + ":", ++n.children.begin(),n.children.end(), state, allow_overwrite);
-
+ auto packName = read_string(n[0]);
+ if (packName.front() != ':') packName = ":" + packName;
+ return GeneratePackage(resolved, packName + ":", ++n.children.begin(), n.children.end(), state);
} },
{ tag::import,[&](const AstNode& n) -> Err<void> {
- package_uri import_uri;
if (n[0].strend) {
- import_uri = {
- "#core",
- "",
- read_string(n[0]) + ".k"
- };
+ return importHook(BufferKey{ "#CORE", "", n[0].get_string() + ".k" });
} else if (n[0].strbeg == tag::lstring) {
- import_uri = uri;
- import_uri.identifier = read_children(n[0]);
- if (import_uri.uri == "#imm") import_uri.uri = "#core";
+ return importHook(BufferKey{ "#LOCAL", "", read_children(n[0]) });
} else if (n[0].strbeg == tag::package) {
- std::string version = n[0][1].get_string();
std::string file = "main.k";
+
if (n[0].children.size() > 2) {
file = n[0][2].get_string();
}
- import_uri = {
- n[0][0].get_string(),
- version,
- file
- };
+
+ return importHook(
+ BufferKey{
+ n[0][0].get_string(),
+ n[0][1].get_string(),
+ file
+ }
+ );
}
- std::string path = import_uri.identifier;
- if (import_uri.uri.size())
- path = TLS::GetCurrentInstance()->ResolveModulePath(import_uri.uri.c_str(), import_uri.identifier.c_str(), import_uri.version.c_str());
- return import_file(import_uri, state, path,false);
+ return { };
} },
{ tag::immediate,[&](const AstNode& n) -> Err<void> {
expr immed{state,&scope};
@@ 1029,81 861,8 @@ namespace K3 {
} }
}));
- if (err.err) {
- revert(changes);
- return err;
- }
-
- auto err2 = emplace_syms(changes, state.namespaces.front(), scope.scope_symbols, changed_symbols);
- if (err2.err) {
- revert(changes);
- }
- return err2;
- }
-
- Err<void> Repository::import_file(const package_uri& uri, parser_state_t& state, const std::string& path, bool allowRedefine) {
- std::stringstream ss;
- std::ifstream file(utf8filename(path));
- if (file.is_open()) {
- ss << file.rdbuf();
- // stop cyclical imports by checking for changes
- if (module_resources[path] != ss.str()) {
- bool is_update = !module_resources[path].empty();
- module_resources[path] = ss.str();
- return import_buffer(uri, state, path, module_resources[path], [](const char*, CGRef) {}, is_update || allowRedefine);
- }
- return {};
- } else {
- return FileError(std::string("Can't open file '") + path + "'");
- }
- }
-
- void Repository::get_position(const char* pos, std::string& uri, int& line, int& column, std::string* show_line) {
- auto module = module_lookup.upper_bound(pos);
- if (module != module_lookup.begin()) {
- --module;
- auto& module_str = module->second->second;
- if (pos >= module_str.data() && pos < module_str.data() + module_str.size()) {
- line = 0; column = 0;
- SeekPosition(module_str.data(), pos, line, column, show_line);
- uri = module->second->first;
- return;
- }
- }
- uri = ""; line = column = 0;
- }
-
- void changeset_t::def(const std::string& qn, CGRef value) {
- rep.symbols[qn][sequence].expr.emplace_back(value);
- }
-
- void changeset_t::defn(const std::string& qn, CGRef value, const std::string& args, const std::string& doc) {
- auto &sym(rep.symbols[qn][sequence]);
- if(doc.size()) sym.metadata[args] += doc;
- sym.expr.emplace_back(Invariant::Constant::New(pack_fn(qn.c_str(),Type(value))));
- sym.is_defn = true;
- }
-
- Err<void> RepositoryBuilder::AddMacro(const char* relative, CGRef val, bool thing) {
- std::string sym = path + relative;
- if (cs.rep.lookup(sym)) {
- return Error::ParserError(Error::RedefinitionNotAllowed, "Redefinition of symbol '" + std::string(sym) + "' is not allowed", nullptr);
- }
- cs.def(sym, val);
- return {};
- }
-
- void RepositoryBuilder::AddFunction(const char* relative, CGRef val, const char *args, const char *doc, const char *fallback) {
- std::string sym = path + relative;
- if (fallback) {
- AddFunction(relative, Evaluate::New(fallback,
- Lib::Reference::New({ fallback }),
- GenericPair::New(Lib::Reference::New({ sym }),
- GenericPair::New(Invariant::Constant::New(Type(relative)),
- GenericArgument::New()))));
- }
-
- cs.defn(sym, val, std::string("(") + args + ")", doc);
+ CHECK_ERR(Reify(":", resolved, defs));
+ return resolved;
}
static std::ostream& json_string(std::ostream& strm, const std::string& str) {
@@ 1128,7 887,56 @@ namespace K3 {
return strm;
}
- void Repository::dump_kernel_functions(std::ostream& faux_kernel) {
+ void Repository2::ExportMetadata(std::ostream& json_doc) {
+ json_doc << "{";
+ bool firstSym = true;
+ for (auto& sym : completeDefinitions) {
+
+ std::stringstream json;
+ json_string(json, sym.first) << ": {";
+
+ bool has_docs = false;
+
+ bool first = true;
+ for (auto& md : sym.second.metadata) {
+ if (md.second.size()) {
+ has_docs = true;
+ if (first) first = false;
+ else json << ", ";
+
+ json_string(json, md.first) << ": ";
+ json_string(json, md.second);
+ }
+ }
+ json << "}";
+
+ if (has_docs) {
+ if (!firstSym) json_doc << ",\n";
+ else firstSym = false;
+
+ json_doc << json.rdbuf();
+ }
+ }
+
+ // dump special forms
+ for (auto b : builtins) {
+ auto md = builtin_metadata.equal_range(b.first);
+ json_doc << ",\n";
+ json_string(json_doc, ":" + b.first) << ": {";
+ for (auto i = md.first; i != md.second; ++i) {
+ if (i != md.first) json_doc << ", ";
+ json_string(json_doc, i->second.first) << ":";
+ json_string(json_doc, i->second.second);
+ }
+ json_doc << "}";
+
+ }
+
+
+ json_doc << "}\n";
+ }
+
+/* void Repository::dump_kernel_functions(std::ostream& faux_kernel) {
std::set<int> kernelChangesets;
for (auto &cs : changesets) {
if (cs.first == "#kernel") {
@@ 1197,63 1005,6 @@ namespace K3 {
<< "\n\tRaise(\"no implementation\")\n}\n\n";
}
}
- }
-
- void Repository::export_metadata(std::ostream& json_doc) {
- json_doc << "{";
- bool firstSym = true;
- for(auto &sym : symbols) {
-
- std::stringstream json;
- json_string(json, sym.first) << ": {";
-
- bool has_docs = false;
-
- std::unordered_map<std::string, std::string> metadata;
- for (auto &change : sym.second) {
- for (auto& doc : change.second.metadata) {
- metadata[doc.first] += doc.second;
- }
- }
-
- bool first = true;
- for (auto& md : metadata) {
- if (md.second.size()) {
- has_docs = true;
- if (first) first = false;
- else json << ", ";
-
- json_string(json, md.first) << ": ";
- json_string(json, md.second);
- }
- }
- json << "}";
-
- if (has_docs) {
- if (!firstSym) json_doc << ",\n";
- else firstSym = false;
-
- json_doc << json.rdbuf();
- }
- }
-
- // dump special forms
- for (auto b : builtins) {
- auto md = builtin_metadata.equal_range(b.first);
- json_doc << ",\n";
- json_string(json_doc, ":" + b.first) << ": {";
- for (auto i = md.first;i != md.second;++i) {
- if (i != md.first) json_doc << ", ";
- json_string(json_doc, i->second.first) << ":";
- json_string(json_doc, i->second.second);
- }
- json_doc << "}";
-
- }
-
-
- json_doc << "}\n";
- }
+ }*/
}
-}
-
+}
No newline at end of file
M src/k3/Parser.h +147 -74
@@ 6,14 6,17 @@
#include "UserErrors.h"
#include "Graph.h"
#include "NodeBases.h"
+#include "RepositoryBuilder.h"
+
#include <string>
-#include <map>
+#include <map>
#include <unordered_map>
#include <deque>
#include <list>
#include <functional>
#include <stdexcept>
#include <vector>
+#include <list>
namespace lithe {
struct node;
@@ 85,107 88,177 @@ namespace K3 {
};
extern DynamicScope<const char*> CurrentSourcePosition;
+
+ using namespace K3::Nodes;
+
+ enum class Attributes {
+ None = 0,
+ Pattern = 1,
+ Extend = 2,
+ AlwaysOverride = 4,
+ MayOverride = 8
+ };
+
+ struct Form {
+ std::unordered_map<std::string, std::string> metadata;
+ CGRef body;
+ Attributes attr = Attributes::None;
+
+ bool HasAttr(Attributes a) const {
+ return ((int)attr & (int)a) != 0;
+ }
+
+ enum Mode {
+ Undefined,
+ Macro,
+ Function,
+ } mode = Undefined;
+
+ static Form Fn(CGRef expr);
+ static Form Val(CGRef expr);
+ };
+
+ class Repository2;
+
struct symbol_t {
Graph<Nodes::Generic> graph;
std::unordered_map<std::string, std::string> metadata;
};
- struct package_uri {
- std::string uri;
- std::string version;
- std::string identifier;
- bool operator==(const package_uri& rhs) const {
- return uri == rhs.uri && version == rhs.version;
- }
-
- enum ordering {
- less,
- unordered,
- equal,
- greater
- };
-
- ordering ordinal_compare(const package_uri& rhs) const {
- if (uri == rhs.uri) {
- if (version == rhs.version) return equal;
- if (version < rhs.version) return less;
- return greater;
- }
- return unordered;
- }
+ struct PartialDefinition {
+ std::vector<Form> forms;
+ std::vector<Type> recurData;
+ Err<CGRef> Complete(Repository2*, const std::string& name = "") const;
+ Err<symbol_t> CompleteMeta (Repository2*, const std::string& name = "") const;
+ static PartialDefinition Value(CGRef expr);
+ void Append(const PartialDefinition& other);
};
struct parser_state_t {
std::deque<std::string> namespaces = { ":" };
std::unordered_map<std::string, std::string> explicit_uses;
};
-
- class Repository;
+
+ using CI = std::vector<AstNode>::const_iterator;
+ using resolution_t = std::pair<const AstNode*, std::function<Err<PartialDefinition>()>>;
+ using resolver_map_t = std::unordered_map<std::string, std::vector<resolution_t>>;
- struct changeset_t {
- int sequence;
- package_uri source;
- bool allow_overwrite;
- Repository& rep;
- void defn(const std::string& qualified_name, Nodes::CGRef expr, const std::string& args, const std::string& doc);
- void def(const std::string& qualified_name, Nodes::CGRef expr);
- };
-
- using uri_changeset_map_t = std::unordered_multimap<std::string, changeset_t>;
- using changeset_ref = uri_changeset_map_t::iterator;
+ struct BufferKey {
+ std::string package, version, path;
+ operator std::string() const {
+ if (path == "main.k") {
+ return "[" + package + " " + version + "]";
+ } else {
+ return "[" + package + " " + version + " " + path + "]";
+ }
+ }
- struct change_t {
- std::unordered_map<std::string, std::string> metadata;
- std::vector<Graph<Nodes::Generic>> expr;
- bool is_defn = false;
- bool overwrite = false;
+ std::string str() const {
+ return *this;
+ }
};
- using symboltable_t = std::unordered_map<std::string, std::map<int, change_t>>;
+ using DefinitionMap = std::unordered_map<std::string, PartialDefinition>;
+
+ Err<DefinitionMap> GenerateSymbols(const AstNode& parseTree,
+ parser_state_t& state,
+ std::function<Err<void>(BufferKey)> importHook,
+ immediate_handler_t imm);
+
+ struct RepositoryNode {
+ Ref<MemoryRegion> graphs = new MemoryRegion;
+ std::unordered_map<std::string, PartialDefinition> changes;
+
+ std::vector<RepositoryNode*> imports;
- using CI = std::vector<AstNode>::const_iterator;
- using resolution_t = std::pair<const AstNode&, std::function<Err<Nodes::CGRef>()>>;
- using resolver_map_t = std::unordered_map<std::string, std::vector<resolution_t>>;
+ std::string retainSource;
+ bool canOverwrite;
+
+ PartialDefinition Resolve(const std::string& qualifiedName, std::unordered_set<RepositoryNode*>& visited);
+
+ void AddImport(RepositoryNode*);
+ void Reset();
- class Repository {
- friend struct changeset_t;
- using module_map_t = std::unordered_map<std::string, std::string>;
+ parser_state_t incremental;
+ BufferKey URI;
+ std::string fileSystemPath;
+ };
+
+ struct RepositoryModule {
+ std::unordered_map<std::string, RepositoryNode> files;
+ std::string version;
+ };
- module_map_t module_resources;
- std::map<const char*, module_map_t::const_iterator> module_lookup;
-
- uri_changeset_map_t changesets;
- int changeset_sequence = 1;
+ class Repository2 {
+ friend struct RepositoryBuilder;
+ Ref<MemoryRegion> stubs = new MemoryRegion();
+ std::unordered_map<std::string, RepositoryModule> modules;
+ std::unordered_map<std::string, symbol_t> completeDefinitions;
+ std::unordered_map<RepositoryNode*, RepositoryNode> rollback;
+ std::list<RepositoryNode> adhoc;
+ RepositoryNode root;
+ RepositoryNode kernel;
+ Err<symbol_t> Build(const std::string& qualifiedName);
+ std::string defaultRepo, defaultVersion;
- symboltable_t symbols;
-
- Err<void> generate_package(changeset_ref changes, std::string name, const CI& beg, const CI& end, parser_state_t state, bool allow_overwrite);
+ struct ImportTask {
+ BufferKey uri;
+ RepositoryNode* node;
+ immediate_handler_t imm;
+ std::string contextPath;
+ };
+
+ std::vector<ImportTask> importQueue;
- Err<void> emplace_syms(changeset_ref changes, const std::string& ns, const resolver_map_t& defs, std::unordered_set<std::string>& touches);
+ Err<void> ImportTree(BufferKey firstLeaf, immediate_handler_t);
+ Err<void> CreateImportTask(RepositoryNode* importer, BufferKey identifier, immediate_handler_t);
+ Err<void> ParseIntoNode(RepositoryNode* node, const std::string& sourceCode, immediate_handler_t);
+ Err<void> Perform(ImportTask);
+
+ std::string relativeBasePath;
- std::unordered_map<std::string, symbol_t> symbol_cache;
+ Err<void> PerformQueue();
+
+ void InvalidateSymbolsInNode(RepositoryNode*);
+
+ Err<void> Transaction(std::function<Err<void>()>);
+
public:
std::unordered_set<std::string> changed_symbols;
- Repository() {}
+ Repository2();
+
+ const symbol_t* Lookup(const std::string& qualifiedName);
+ void Rebind(const std::string& qualifiedName, Nodes::CGRef expr);
- symbol_t build_symbol(const std::string& qualified_name);
- const symbol_t* lookup(const std::string& qualified_name);
- void temporary_rebind(const std::string& qualified_name, Nodes::CGRef expr);
+ ParserError RedefinitionError(std::string qn,
+ const char* newPoint, Form::Mode newMode,
+ const char* oldPoint, Form::Mode oldMode);
- const symboltable_t& get_symboltable() const { return symbols; }
+ void SetCoreLib(std::string repo, std::string version) {
+ defaultRepo = repo;
+ defaultVersion = version;
+ }
- Err<changeset_ref> new_changeset(const package_uri& src, const std::string& file, bool can_overwrite = false);
- void revert(changeset_ref);
+ const std::string& GetCoreLibPackage() {
+ return defaultRepo;
+ }
+
+ const std::string& GetCoreLibVersion() {
+ return defaultVersion;
+ }
+
+ void ExportMetadata(std::ostream& json);
- Err<void> import_file(const package_uri& pack_uri, parser_state_t& state, const std::string& path, bool allow_overwrite);
-
- Err<void> import_buffer(const package_uri& pack_uri, parser_state_t& state, const std::string& id, const std::string& text, immediate_handler_t, bool allow_overwrite);
-
- void get_position(const char* mem_pos, std::string& uri, int& line, int& column, std::string* show_line);
- void export_metadata(std::ostream& json);
+ Err<void> ImportFile(const char* path, bool canOverwrite);
+ Err<void> ImportBuffer(const char* code, bool canOverwrite, immediate_handler_t = {});
+ Err<void> ImportCoreLib(const char* file);
- void dump_kernel_functions(std::ostream&);
+ RepositoryBuilder GetKernelBuilder();
+
+ void GetPosition(const char* mem_pos, std::string& uri, int& line, int& column, std::string* show_line);
+
+ Err<void> UpdateDefinitions();
};
};
-};
+};
No newline at end of file
M src/k3/Reactive.cpp +0 -1
@@ 1077,7 1077,6 @@ namespace K3 {
void BuildReactivePrimitiveOps(Parser::RepositoryBuilder pack) {
using namespace Nodes;
using namespace Nodes::ReactiveOperators;
- RegionAllocator PrimitiveOpAllocator;
auto arg = GenericArgument::New();
auto b1 = GenericFirst::New(arg);
auto b2 = GenericRest::New(arg);
M src/k3/RepositoryBuilder.h +8 -6
@@ 4,16 4,18 @@
namespace K3 {
namespace Parser {
- class Repository;
- struct changeset_t;
+ class Repository2;
+
struct RepositoryBuilder {
std::string path;
- changeset_t& cs;
- Err<void> AddMacro(const char*, Nodes::CGRef, bool);
- void AddFunction(const char*, Nodes::CGRef, const char* args="", const char *docs = "", const char*fallback = nullptr);
+ Repository2& r;
+ void AddMacro(const char*, Nodes::CGRef, bool);
+ void AddFunction(const char*, Nodes::CGRef, const char* args="", const char *docs = "", const char *fallback = nullptr);
RepositoryBuilder AddPackage(const char* name) {
- return RepositoryBuilder{ path + name + ":", cs };
+ return RepositoryBuilder{ path + name + ":", r };
}
+ ~RepositoryBuilder();
+ MemoryRegion* GetMemoryRegion();
};
}
}
M src/k3/TLS.cpp +13 -17
@@ 35,8 35,8 @@ namespace K3 {
void BuildSelectPrimitiveOps(Parser::RepositoryBuilder);
void BuildNativePrimitiveOps(Parser::RepositoryBuilder);
- void BuildKernelFunctions(Parser::RepositoryBuilder build) {
- RegionAllocator KernelBaseAllocator;
+ void BuildKernelFunctions(Parser::RepositoryBuilder& build) {
+ RegionAllocator KernelBaseAllocator{ build.GetMemoryRegion() };
auto arg = Nodes::GenericArgument::New();
auto b1 = Nodes::GenericFirst::New(arg);
@@ 166,19 166,15 @@ namespace K3 {
Err<void> TLS::Initialize() {
ScopedContext cx(*this);
- LET_ERR(kernel, codebase.new_changeset(Parser::package_uri{ "#kernel" }, "#builtin", false));
- BuildKernelFunctions(Parser::RepositoryBuilder{":", kernel->second});
+ auto builder{ codebase.GetKernelBuilder() };
+ BuildKernelFunctions(builder);
+ codebase.UpdateDefinitions();
// dump the kernel function definitions by uncommenting this
// codebase.dump_kernel_functions(std::cout);
- Parser::parser_state_t tmpScope;
- Parser::package_uri prelude {
- "#core",
- "",
- "Prelude.k"
- };
- return codebase.import_file(prelude, tmpScope, ResolveModulePath("#core", "Prelude.k", ""), false);
+ // parse prelude
+ return codebase.ImportCoreLib("Prelude.k");
}
TLS::TLS(Kronos::ModulePathResolver res, void *user) {
@@ 194,7 190,7 @@ namespace K3 {
if (err.err->GetSourceFilePosition()) {
std::string url, show;
int line, column;
- codebase.get_position(err.err->GetSourceFilePosition(), url, line, column, &show);
+ codebase.GetPosition(err.err->GetSourceFilePosition(), url, line, column, &show);
if (url.size()) {
std::cerr << "While preloading " << url << ":(" << line << "," << column << ")\n";
std::cerr << show << "\n";
@@ 232,14 228,14 @@ namespace K3 {
Nodes::CGRef TLS::ResolveSymbol(const char* qualifiedName) {
auto self = TLS::GetCurrentInstance();
- auto sym = self->codebase.lookup(qualifiedName);
+ auto sym = self->codebase.Lookup(qualifiedName);
self->DidResolve(qualifiedName);
return sym ? sym->graph : nullptr;
}
void TLS::RebindSymbol(const char *qualifiedName, Nodes::CGRef graph) {
auto self = TLS::GetCurrentInstance();
- self->codebase.temporary_rebind(qualifiedName, graph);
+ self->codebase.Rebind(qualifiedName, graph);
}
std::unordered_set<std::string> TLS::DrainRecentChanges() {
@@ 252,7 248,7 @@ namespace K3 {
std::stringstream text;
int line, column;
std::string uri;
- TLS::GetCurrentInstance()->codebase.get_position(sourcePos, uri, line, column, showLine);
+ TLS::GetCurrentInstance()->codebase.GetPosition(sourcePos, uri, line, column, showLine);
if (uri.empty()) return uri;
text << uri << "(" << line << ";" << column << ")";
return text.str();
@@ 260,7 256,7 @@ namespace K3 {
Type TLS::GetRepositoryMetadata(const std::string& prefix) {
Type lib;
- for (auto &sym : GetCurrentInstance()->codebase.get_symboltable()) {
+/* for (auto &sym : GetCurrentInstance()->codebase.get_symboltable()) {
if (sym.first.size() >= prefix.size()) {
for (size_t i(0);i < prefix.size();++i) {
if (sym.first[i] != prefix[i]) goto next;
@@ 274,7 270,7 @@ namespace K3 {
}
}
next:;
- };
+ };*/
return lib;
}
M src/k3/TLS.h +1 -1
@@ 71,7 71,7 @@ namespace K3 {
std::unordered_set<std::string> GetResolutionTrace() const { return resolutionTrace; }
std::unordered_set<std::string> DrainRecentChanges();
Parser::parser_state_t REPLState;
- Parser::Repository codebase;
+ Parser::Repository2 codebase;
void InitializeDefaultResolver();
public:
M src/k3/kronos.cpp +20 -38
@@ 243,6 243,18 @@ namespace {
ContextImpl(ModulePathResolver res, void *user):TLS(res, user) {
}
+ void _SetDefaultRepository(const char* package, const char* version) override {
+ codebase.SetCoreLib(package, version);
+ }
+
+ const char* _GetCoreLibPackage() override {
+ return codebase.GetCoreLibPackage().c_str();
+ }
+
+ const char* _GetCoreLibVersion() override {
+ return codebase.GetCoreLibVersion().c_str();
+ }
+
void _SetCompilerDebugTraceFilter(const char *flt) noexcept override {
SetCompilerDebugTraceFilter(flt);
}
@@ 251,19 263,9 @@ namespace {
XX([&](){
ScopedContext scope(*this);
RegionAllocator parsedNodes;
- std::stringstream time;
- auto t = std::time(nullptr);
- static int counter = 0;
-#if defined(__GNUC__) && __GNUC__ < 5
- time << t <<"("<<counter++<<")";
-#else
- time << std::put_time(std::localtime(&t), "#%F:%T") << "(" << counter++ << ")";
-#endif
- auto parsed = codebase.import_buffer({"#imm","",time.str()}, REPLState, time.str(), source, [handler, userdata](const char* sym, CGRef imm) mutable {
+ return codebase.ImportBuffer(source, true, [handler, userdata](const char* sym, CGRef imm) mutable {
handler(userdata, sym, Ref<GenericGraphImpl>::Cons(imm));
- }, true);
-
- return parsed;
+ });
});
}
@@ 303,25 305,6 @@ namespace {
return ptr;
});
}
-
- IStr* _GetJsonCompletionItems(const char *file, int row, int column) const noexcept override {
- return XX([&]() {
- std::stringstream items;
- bool first = true;
- items << "[";
- for (const auto& sym : codebase.get_symboltable()) {
- if (first) first = false;
- else items << ",";
- items << "{"
- "\"label\": \"" << sym.first << "\""
- "}";
- }
-
- items << "]";
-
- return new _String(items.str());
- });
- }
virtual const ITypedGraph* _Specialize(const IGenericGraph* GAST, const IType& argument, IStreamBuf* _log, int logLevel) noexcept override {
return XX([&]() -> Err<ITypedGraph*> {
@@ 354,7 337,7 @@ namespace {
return XX([&]() {
_Streambuf jsonBuf(buf);
std::ostream json(&jsonBuf);
- codebase.export_metadata(json);
+ codebase.ExportMetadata(json);
return 1;
});
}
@@ 443,15 426,14 @@ namespace {
virtual void _ImportFile(const char *modulePath, KRONOS_INT allowRedefine) noexcept override {
XX([&]() {
SetForThisThread();
- Parser::parser_state_t tmpScope;
- return codebase.import_file({"#core", "", modulePath}, tmpScope, modulePath, allowRedefine != 0);
+ return codebase.ImportFile(modulePath, allowRedefine != 0);
});
}
- virtual void _ImportBuffer(const char* module, const char* sourceCode, const char* inPackage, bool allowRedefinition) noexcept override {
+ virtual void _ImportBuffer(const char* sourceCode, bool allowRedefinition) noexcept override {
XX([&]() {
SetForThisThread();
- return codebase.import_buffer({ "#core", "", inPackage }, REPLState, module, sourceCode, [](const char*, Nodes::CGRef) {}, allowRedefinition);
+ return codebase.ImportBuffer(sourceCode, allowRedefinition);
});
}
@@ 486,7 468,7 @@ namespace {
}
virtual void _BindConstant(const char* name, const IType* ty, const void* data) override {
- RegionAllocator alloc;
+/* RegionAllocator alloc;
SetForThisThread();
auto val = Invariant::Constant::New(ty->GetPimpl(), data);
auto cs = codebase.new_changeset(Parser::package_uri{
@@ 494,7 476,7 @@ namespace {
"",
name
}, name, true);
- if (!cs.err) (*cs)->second.def(name, val);
+ if (!cs.err) (*cs)->second.def(name, val);*/
}
};
}
M src/kronos.h +8 -2
@@ 312,12 312,18 @@ namespace Kronos {
}
Context(IContext *ic = nullptr) : Shared<IContext>(ic) { }
+
+ inline void SetCoreLibrary(const char* repository, const char* version) { Get()->_SetDefaultRepository(repository, version); }
+ inline void GetCoreLibrary(std::string& repo, std::string& version) {
+ repo = Get()->_GetCoreLibPackage();
+ version = Get()->_GetCoreLibVersion();
+ }
+
inline std::string GetModuleAndLineNumberText(const char* offset) { return MoveString(_CheckResult(Get()->_GetModuleAndLineNumberText(offset))); }
inline std::string ShowModuleLine(const char* offset) { return MoveString(_CheckResult(Get()->_ShowModuleLine(offset))); }
- inline std::string GetCompletionItems(const char *path, int row, int column) { return MoveString(_CheckResult(Get()->_GetJsonCompletionItems(path, row, column))); }
inline void ImportFile(const std::string& modulePath, bool allowRedefinition = false) { Get()->_ImportFile(modulePath.c_str(), allowRedefinition ? 1 : 0); _CheckLastError(); }
- inline void ImportBuffer(const std::string& module, const std::string& sourceCode, const std::string& packageId = "", bool allowRedefinition = false) { Get( )->_ImportBuffer(module.c_str( ), sourceCode.c_str( ), packageId.c_str( ), allowRedefinition); _CheckLastError(); }
+ inline void ImportBuffer(const std::string& sourceCode, bool allowRedefinition = false) { Get( )->_ImportBuffer(sourceCode.c_str( ), allowRedefinition); _CheckLastError(); }
inline Type TypeFromUID(std::int64_t uid) {
return Get()->_TypeFromUID(uid);
M src/kronos_abi.h +4 -2
@@ 244,10 244,9 @@ namespace Kronos {
class IContext : public ICompilerInterface {
public:
virtual void MEMBER _ImportFile(const char *modulePath, INT allowRedefine) noexcept = 0;
- virtual void MEMBER _ImportBuffer(const char* module, const char* sourceCode, const char* inPackage, bool allowRedefinition) noexcept = 0;
+ virtual void MEMBER _ImportBuffer(const char* sourceCode, bool allowRedefinition) noexcept = 0;
virtual IStr* MEMBER _GetModuleAndLineNumberText(const char*) noexcept = 0;
virtual IStr* MEMBER _ShowModuleLine(const char*) noexcept = 0;
- virtual IStr* MEMBER _GetJsonCompletionItems(const char *file, int row, int column) const noexcept = 0;
virtual INT MEMBER _GetLibraryMetadataAsJSON(IStreamBuf* json) = 0;
virtual IType* MEMBER _TypeFromUID(std::int64_t uid) noexcept = 0;
virtual std::int64_t MEMBER _UIDFromType(const IType*) noexcept = 0;
@@ 256,6 255,9 @@ namespace Kronos {
virtual void MEMBER SetAssetLinker(AssetLinker al, void* user) noexcept = 0;
virtual void MEMBER _BindConstant(const char*, const IType*, const void*) = 0;
virtual void MEMBER _SetCompilerDebugTraceFilter(const char *) = 0;
+ virtual void MEMBER _SetDefaultRepository(const char* package, const char* version) = 0;
+ virtual const char* MEMBER _GetCoreLibPackage() = 0;
+ virtual const char* MEMBER _GetCoreLibVersion() = 0;
};
ABI const char* FUNCTION GetVersionString( ) noexcept;
M src/website/mod2json.cpp +1 -1
@@ 39,7 39,7 @@ void read_package(std::string path, lith
}
int compile_module(const char *mod, const char *path) {
- Packages::DefaultClient bbClient("", "");
+ Packages::DefaultClient bbClient;
auto grammar = lithe::grammar::kronos::parser();
std::string mname = mod;