Welcome to TiddlyWiki created by Jeremy Ruston, Copyright © 2007 UnaMesa Association
!!About skel
I originally wrote skel during the summer of 2004 while working at lastminute.com. It was never used for its original purpose as the project it was for was cancelled. Since that time I've used it extensively to configure environments on various Unix and Unix-like paltforms. It should run on any reasonably recent Unix-like system: I currently use it on Mac OS X and Solaris 10, but have used it on several Linux platforms as well.
!!About this manual
The original skel manual was written using [[TWiki|http://twiki.org/]]. This version has been converted to Jeremy Ruston's excellent [[TiddlyWiki|http://www.tiddlywiki.org/]] system, as an experiment to see if it was suitable for documention of this kind - on the whole I think it isn't, although it is a wonderful note-taking system. The figures were produced with [[OmniGraffle Professional|http://www.omnigroup.com/applications/omnigraffle/]].
This manual is far from being complete or coherent, and has a number of significant bugs and omissions. I hope to improve things in a future version.
!!Acknowledgements
Thanks to lastminute.com for allowing me to open source skel, and specifically to Chris Gathercole for arranging things.
!!Bugs and improvements
Please contact me, [[Tim Bradshaw]] with any bugs or suggestions for improvements.
!!License information
<<tiddler License>>
----
{{{$Id: //depot/www-tfeb-org/main/www-tfeb-org/html/programs/skel/Skel.html#2 $}}}
!!In skel
* If the environment file has no non-null lines the wrapper falls about because it assumes it failed to read it. It is safe (and perhaps good practice) to have a line saying <html>
<pre>require ROOT
</pre></html> but it should not be mandatory.
/***
|''Name:''|CryptoFunctionsPlugin|
|''Description:''|Support for cryptographic functions|
***/
//{{{
if(!version.extensions.CryptoFunctionsPlugin) {
version.extensions.CryptoFunctionsPlugin = {installed:true};
//--
//-- Crypto functions and associated conversion routines
//--
// Crypto "namespace"
function Crypto() {}
// Convert a string to an array of big-endian 32-bit words
Crypto.strToBe32s = function(str)
{
var be = Array();
var len = Math.floor(str.length/4);
var i, j;
for(i=0, j=0; i<len; i++, j+=4) {
be[i] = ((str.charCodeAt(j)&0xff) << 24)|((str.charCodeAt(j+1)&0xff) << 16)|((str.charCodeAt(j+2)&0xff) << 8)|(str.charCodeAt(j+3)&0xff);
}
while (j<str.length) {
be[j>>2] |= (str.charCodeAt(j)&0xff)<<(24-(j*8)%32);
j++;
}
return be;
};
// Convert an array of big-endian 32-bit words to a string
Crypto.be32sToStr = function(be)
{
var str = "";
for(var i=0;i<be.length*32;i+=8)
str += String.fromCharCode((be[i>>5]>>>(24-i%32)) & 0xff);
return str;
};
// Convert an array of big-endian 32-bit words to a hex string
Crypto.be32sToHex = function(be)
{
var hex = "0123456789ABCDEF";
var str = "";
for(var i=0;i<be.length*4;i++)
str += hex.charAt((be[i>>2]>>((3-i%4)*8+4))&0xF) + hex.charAt((be[i>>2]>>((3-i%4)*8))&0xF);
return str;
};
// Return, in hex, the SHA-1 hash of a string
Crypto.hexSha1Str = function(str)
{
return Crypto.be32sToHex(Crypto.sha1Str(str));
};
// Return the SHA-1 hash of a string
Crypto.sha1Str = function(str)
{
return Crypto.sha1(Crypto.strToBe32s(str),str.length);
};
// Calculate the SHA-1 hash of an array of blen bytes of big-endian 32-bit words
Crypto.sha1 = function(x,blen)
{
// Add 32-bit integers, wrapping at 32 bits
add32 = function(a,b)
{
var lsw = (a&0xFFFF)+(b&0xFFFF);
var msw = (a>>16)+(b>>16)+(lsw>>16);
return (msw<<16)|(lsw&0xFFFF);
};
// Add five 32-bit integers, wrapping at 32 bits
add32x5 = function(a,b,c,d,e)
{
var lsw = (a&0xFFFF)+(b&0xFFFF)+(c&0xFFFF)+(d&0xFFFF)+(e&0xFFFF);
var msw = (a>>16)+(b>>16)+(c>>16)+(d>>16)+(e>>16)+(lsw>>16);
return (msw<<16)|(lsw&0xFFFF);
};
// Bitwise rotate left a 32-bit integer by 1 bit
rol32 = function(n)
{
return (n>>>31)|(n<<1);
};
var len = blen*8;
// Append padding so length in bits is 448 mod 512
x[len>>5] |= 0x80 << (24-len%32);
// Append length
x[((len+64>>9)<<4)+15] = len;
var w = Array(80);
var k1 = 0x5A827999;
var k2 = 0x6ED9EBA1;
var k3 = 0x8F1BBCDC;
var k4 = 0xCA62C1D6;
var h0 = 0x67452301;
var h1 = 0xEFCDAB89;
var h2 = 0x98BADCFE;
var h3 = 0x10325476;
var h4 = 0xC3D2E1F0;
for(var i=0;i<x.length;i+=16) {
var j,t;
var a = h0;
var b = h1;
var c = h2;
var d = h3;
var e = h4;
for(j = 0;j<16;j++) {
w[j] = x[i+j];
t = add32x5(e,(a>>>27)|(a<<5),d^(b&(c^d)),w[j],k1);
e=d; d=c; c=(b>>>2)|(b<<30); b=a; a = t;
}
for(j=16;j<20;j++) {
w[j] = rol32(w[j-3]^w[j-8]^w[j-14]^w[j-16]);
t = add32x5(e,(a>>>27)|(a<<5),d^(b&(c^d)),w[j],k1);
e=d; d=c; c=(b>>>2)|(b<<30); b=a; a = t;
}
for(j=20;j<40;j++) {
w[j] = rol32(w[j-3]^w[j-8]^w[j-14]^w[j-16]);
t = add32x5(e,(a>>>27)|(a<<5),b^c^d,w[j],k2);
e=d; d=c; c=(b>>>2)|(b<<30); b=a; a = t;
}
for(j=40;j<60;j++) {
w[j] = rol32(w[j-3]^w[j-8]^w[j-14]^w[j-16]);
t = add32x5(e,(a>>>27)|(a<<5),(b&c)|(d&(b|c)),w[j],k3);
e=d; d=c; c=(b>>>2)|(b<<30); b=a; a = t;
}
for(j=60;j<80;j++) {
w[j] = rol32(w[j-3]^w[j-8]^w[j-14]^w[j-16]);
t = add32x5(e,(a>>>27)|(a<<5),b^c^d,w[j],k4);
e=d; d=c; c=(b>>>2)|(b<<30); b=a; a = t;
}
h0 = add32(h0,a);
h1 = add32(h1,b);
h2 = add32(h2,c);
h3 = add32(h3,d);
h4 = add32(h4,e);
}
return Array(h0,h1,h2,h3,h4);
};
}
//}}}
/***
|''Name:''|DeprecatedFunctionsPlugin|
|''Description:''|Support for deprecated functions removed from core|
***/
//{{{
if(!version.extensions.DeprecatedFunctionsPlugin) {
version.extensions.DeprecatedFunctionsPlugin = {installed:true};
//--
//-- Deprecated code
//--
// @Deprecated: Use createElementAndWikify and this.termRegExp instead
config.formatterHelpers.charFormatHelper = function(w)
{
w.subWikify(createTiddlyElement(w.output,this.element),this.terminator);
};
// @Deprecated: Use enclosedTextHelper and this.lookaheadRegExp instead
config.formatterHelpers.monospacedByLineHelper = function(w)
{
var lookaheadRegExp = new RegExp(this.lookahead,"mg");
lookaheadRegExp.lastIndex = w.matchStart;
var lookaheadMatch = lookaheadRegExp.exec(w.source);
if(lookaheadMatch && lookaheadMatch.index == w.matchStart) {
var text = lookaheadMatch[1];
if(config.browser.isIE)
text = text.replace(/\n/g,"\r");
createTiddlyElement(w.output,"pre",null,null,text);
w.nextMatch = lookaheadRegExp.lastIndex;
}
};
// @Deprecated: Use <br> or <br /> instead of <<br>>
config.macros.br = {};
config.macros.br.handler = function(place)
{
createTiddlyElement(place,"br");
};
// Find an entry in an array. Returns the array index or null
// @Deprecated: Use indexOf instead
Array.prototype.find = function(item)
{
var i = this.indexOf(item);
return i == -1 ? null : i;
};
// Load a tiddler from an HTML DIV. The caller should make sure to later call Tiddler.changed()
// @Deprecated: Use store.getLoader().internalizeTiddler instead
Tiddler.prototype.loadFromDiv = function(divRef,title)
{
return store.getLoader().internalizeTiddler(store,this,title,divRef);
};
// Format the text for storage in an HTML DIV
// @Deprecated Use store.getSaver().externalizeTiddler instead.
Tiddler.prototype.saveToDiv = function()
{
return store.getSaver().externalizeTiddler(store,this);
};
// @Deprecated: Use store.allTiddlersAsHtml() instead
function allTiddlersAsHtml()
{
return store.allTiddlersAsHtml();
}
// @Deprecated: Use refreshPageTemplate instead
function applyPageTemplate(title)
{
refreshPageTemplate(title);
}
// @Deprecated: Use story.displayTiddlers instead
function displayTiddlers(srcElement,titles,template,unused1,unused2,animate,unused3)
{
story.displayTiddlers(srcElement,titles,template,animate);
}
// @Deprecated: Use story.displayTiddler instead
function displayTiddler(srcElement,title,template,unused1,unused2,animate,unused3)
{
story.displayTiddler(srcElement,title,template,animate);
}
// @Deprecated: Use functions on right hand side directly instead
var createTiddlerPopup = Popup.create;
var scrollToTiddlerPopup = Popup.show;
var hideTiddlerPopup = Popup.remove;
// @Deprecated: Use right hand side directly instead
var regexpBackSlashEn = new RegExp("\\\\n","mg");
var regexpBackSlash = new RegExp("\\\\","mg");
var regexpBackSlashEss = new RegExp("\\\\s","mg");
var regexpNewLine = new RegExp("\n","mg");
var regexpCarriageReturn = new RegExp("\r","mg");
}
//}}}
/***
|Name|HoverMenuPlugin|
|Created by|SaqImtiaz|
|Location|http://lewcid.googlepages.com/lewcid.html#HoverMenuPlugin|
|Version|1.11|
|Requires|~TW2.x|
!Description:
Provides a hovering menu on the edge of the screen for commonly used commands, that scrolls with the page.
!Demo:
Observe the hovering menu on the right edge of the screen.
!Installation:
Copy the contents of this tiddler to your TW, tag with systemConfig, save and reload your TW.
To customize your HoverMenu, edit the HoverMenu shadow tiddler.
To customize whether the menu sticks to the right or left edge of the screen, and its start position, edit the HoverMenu configuration settings part of the code below. It's well documented, so don't be scared!
The menu has an id of hoverMenu, in case you want to style the buttons in it using css.
!Notes:
Since the default HoverMenu contains buttons for toggling the side bar and jumping to the top of the screen and to open tiddlers, the ToggleSideBarMacro, JumpMacro and the JumpToTopMacro are included in this tiddler, so you dont need to install them separately. Having them installed separately as well could lead to complications.
If you dont intend to use these three macros at all, feel free to remove those sections of code in this tiddler.
!To Do:
* rework code to allow multiple hovering menus in different positions, horizontal etc.
* incorporate code for keyboard shortcuts that correspond to the buttons in the hovermenu
!History:
*03-08-06, ver 1.11: fixed error with button tooltips
*27-07-06, ver 1.1 : added JumpMacro to hoverMenu
*23-07-06
!Code
***/
/***
start HoverMenu plugin code
***/
//{{{
config.hoverMenu={};
//}}}
/***
HoverMenu configuration settings
***/
//{{{
config.hoverMenu.settings={
align: 'right', //align menu to right or left side of screen, possible values are 'right' and 'left'
x: 1, // horizontal distance of menu from side of screen, increase to your liking.
y: 158 //vertical distance of menu from top of screen at start, increase or decrease to your liking
};
//}}}
//{{{
//continue HoverMenu plugin code
config.hoverMenu.handler=function()
{
var theMenu = createTiddlyElement(document.getElementById("contentWrapper"), "div","hoverMenu");
theMenu.setAttribute("refresh","content");
theMenu.setAttribute("tiddler","HoverMenu");
var menuContent = store.getTiddlerText("HoverMenu");
wikify(menuContent,theMenu);
var Xloc = this.settings.x;
Yloc =this.settings.y;
var ns = (navigator.appName.indexOf("Netscape") != -1);
function SetMenu(id)
{
var GetElements=document.getElementById?document.getElementById(id):document.all?document.all[id]:document.layers[id];
if(document.layers)GetElements.style=GetElements;
GetElements.sP=function(x,y){this.style[config.hoverMenu.settings.align]=x +"px";this.style.top=y +"px";};
GetElements.x = Xloc;
GetElements.y = findScrollY();
GetElements.y += Yloc;
return GetElements;
}
window.LoCate_XY=function()
{
var pY = findScrollY();
ftlObj.y += (pY + Yloc - ftlObj.y)/15;
ftlObj.sP(ftlObj.x, ftlObj.y);
setTimeout("LoCate_XY()", 10);
}
ftlObj = SetMenu("hoverMenu");
LoCate_XY();
};
window.old_lewcid_hovermenu_restart = restart;
restart = function()
{
window.old_lewcid_hovermenu_restart();
config.hoverMenu.handler();
};
setStylesheet(
"#hoverMenu .button, #hoverMenu .tiddlyLink {border:none; font-weight:bold; background:#18f; color:#FFF; padding:0 5px; float:right; margin-bottom:4px;}\n"+
"#hoverMenu .button:hover, #hoverMenu .tiddlyLink:hover {font-weight:bold; border:none; color:#fff; background:#000; padding:0 5px; float:right; margin-bottom:4px;}\n"+
"#hoverMenu .button {width:100%; text-align:center}"+
"#hoverMenu { position:absolute; width:7px;}\n"+
"\n","hoverMenuStyles");
config.macros.renameButton={};
config.macros.renameButton.handler = function(place,macroName,params,wikifier,paramString,tiddler)
{
if (place.lastChild.tagName!="BR")
{
place.lastChild.firstChild.data = params[0];
if (params[1]) {place.lastChild.title = params[1];}
}
};
config.shadowTiddlers["HoverMenu"]="<<top>>\n<<toggleSideBar>><<renameButton '>' >>\n<<jump j '' top>>\n<<saveChanges>><<renameButton s 'Save TiddlyWiki'>>\n<<newTiddler>><<renameButton n>>\n";
//}}}
//end HoverMenu plugin code
//Start ToggleSideBarMacro code
//{{{
config.macros.toggleSideBar={};
config.macros.toggleSideBar.settings={
styleHide : "#sidebar { display: none;}\n"+"#contentWrapper #displayArea { margin-right: 1em;}\n"+"",
styleShow : " ",
arrow1: "«",
arrow2: "»"
};
config.macros.toggleSideBar.handler=function (place,macroName,params,wikifier,paramString,tiddler)
{
var tooltip= params[1]||'toggle sidebar';
var mode = (params[2] && params[2]=="hide")? "hide":"show";
var arrow = (mode == "hide")? this.settings.arrow1:this.settings.arrow2;
var label= (params[0]&¶ms[0]!='.')?params[0]+" "+arrow:arrow;
var theBtn = createTiddlyButton(place,label,tooltip,this.onToggleSideBar,"button HideSideBarButton");
if (mode == "hide")
{
(document.getElementById("sidebar")).setAttribute("toggle","hide");
setStylesheet(this.settings.styleHide,"ToggleSideBarStyles");
}
};
config.macros.toggleSideBar.onToggleSideBar = function(){
var sidebar = document.getElementById("sidebar");
var settings = config.macros.toggleSideBar.settings;
if (sidebar.getAttribute("toggle")=='hide')
{
setStylesheet(settings.styleShow,"ToggleSideBarStyles");
sidebar.setAttribute("toggle","show");
this.firstChild.data= (this.firstChild.data).replace(settings.arrow1,settings.arrow2);
}
else
{
setStylesheet(settings.styleHide,"ToggleSideBarStyles");
sidebar.setAttribute("toggle","hide");
this.firstChild.data= (this.firstChild.data).replace(settings.arrow2,settings.arrow1);
}
return false;
}
setStylesheet(".HideSideBarButton .button {font-weight:bold; padding: 0 5px;}\n","ToggleSideBarButtonStyles");
//}}}
//end ToggleSideBarMacro code
//start JumpToTopMacro code
//{{{
config.macros.top={};
config.macros.top.handler=function(place,macroName)
{
createTiddlyButton(place,"^","jump to top",this.onclick);
}
config.macros.top.onclick=function()
{
window.scrollTo(0,0);
};
config.commands.top =
{
text:" ^ ",
tooltip:"jump to top"
};
config.commands.top.handler = function(event,src,title)
{
window.scrollTo(0,0);
}
//}}}
//end JumpToStartMacro code
//start JumpMacro code
//{{{
config.macros.jump= {};
config.macros.jump.handler = function (place,macroName,params,wikifier,paramString,tiddler)
{
var label = (params[0] && params[0]!=".")? params[0]: 'jump';
var tooltip = (params[1] && params[1]!=".")? params[1]: 'jump to an open tiddler';
var top = (params[2] && params[2]=='top') ? true: false;
var btn =createTiddlyButton(place,label,tooltip,this.onclick);
if (top==true)
btn.setAttribute("top","true")
}
config.macros.jump.onclick = function(e)
{
if (!e) var e = window.event;
var theTarget = resolveTarget(e);
var top = theTarget.getAttribute("top");
var popup = Popup.create(this);
if(popup)
{
if(top=="true")
{createTiddlyButton(createTiddlyElement(popup,"li"),'Top ↑','Top of TW',config.macros.jump.top);
createTiddlyElement(popup,"hr");}
story.forEachTiddler(function(title,element) {
createTiddlyLink(createTiddlyElement(popup,"li"),title,true);
});
}
Popup.show(popup,false);
e.cancelBubble = true;
if (e.stopPropagation) e.stopPropagation();
return false;
}
config.macros.jump.top = function()
{
window.scrollTo(0,0);
}
//}}}
//end JumpMacro code
//utility functions
//{{{
Popup.show = function(unused,slowly)
{
var curr = Popup.stack[Popup.stack.length-1];
var rootLeft = findPosX(curr.root);
var rootTop = findPosY(curr.root);
var rootHeight = curr.root.offsetHeight;
var popupLeft = rootLeft;
var popupTop = rootTop + rootHeight;
var popupWidth = curr.popup.offsetWidth;
var winWidth = findWindowWidth();
if (isChild(curr.root,'hoverMenu'))
var x = config.hoverMenu.settings.x;
else
var x = 0;
if(popupLeft + popupWidth+x > winWidth)
popupLeft = winWidth - popupWidth -x;
if (isChild(curr.root,'hoverMenu'))
{curr.popup.style.right = x + "px";}
else
curr.popup.style.left = popupLeft + "px";
curr.popup.style.top = popupTop + "px";
curr.popup.style.display = "block";
addClass(curr.root,"highlight");
if(config.options.chkAnimate)
anim.startAnimating(new Scroller(curr.popup,slowly));
else
window.scrollTo(0,ensureVisible(curr.popup));
}
window.isChild = function(e,parentId) {
while (e != null) {
var parent = document.getElementById(parentId);
if (parent == e) return true;
e = e.parentNode;
}
return false;
};
//}}}
!!The problem Skel is trying to solve
<<slider Skel/intro/1 [[Introduction / 1 / The problem Skel is trying to solve]]
/ "what it's trying to do">>
!!A specific deployment example: ~PiSearch
<<slider Skel/intro/2 [[Introduction / 2 / A specific deployment example: PiSearch]]
/ "the original application skel was designed for">>
!!Some approaches that won't work
<<slider Skel/intro/3 [[Introduction / 3 / Some approaches that won't work]]
/ "it's not trivial">>
!!Some approaches that could work
<<slider Skel/intro/4 [[Introduction / 4 / Some approaches that could work]]
/ "some things we could do">>
Very few applications are truly standalone, in the sense that they rely only on the hardware to run. On Unix, even statically-linked binaries rely on the services provided by the kernel. The kernel itself is generally almost standalone - it may rely on other code to load it into memory and do some initial configuration, but after that it runs directly on the hardware. Most applications need libraries of various kinds as well as other applications and suitable configuration & other data files to run, and all of these need to be in compatible versions.
In an ideal world, suitable support for an application would already be installed on the target platform, in the right versions, and everything would be easy. In an ideal world, people would design software to be compatible, so version skew wasn't such a problem. But unfortunately the real world won't arrive until sometime in the late 20^^th^^ century and we're still living in the dark ages, so we have a thousand years to wait. That sound you can hear in the background is people lurching around in the mud and occasionally chopping bits off other people for belonging to the wrong cult, all the while going on endlessly about how smart they are. Unfortunately, we need to install software in this world.
Skel is a tool that supports installing and configuring software on Unix platforms in these dark ages.
The general situation that skel tries to solve is as follows.
* We want to deploy an application //A//, which we've written or obtained, on a target platform //S// (`//S//' for `system'). //A// relies on a number of support libraries and other bits of software, which we will call `packages'. From skel's point of view, a package is just a collection of files - more on this below. We'll call these packages //P~~i~~//, so specifically //P~~1~~// and //P~~2~~//, say.
* All of //A//, the various //P//s, and //S// have version information associated with them - in fact //S// will typically have a very large amount of version information (kernel //x.y//, patch //z//, gcc //a.b// and so on). At least //S// will also have some configuration information associated with it - how the system was set up and so on. There will be all sorts of compatibility constraints between versions and configurations - `//x// will only work with //y// versions greater than 2.83; //z// must also be installed, and you need patch 1273 for //q//' and so on. Somehow, we need to resolve all these compatibility issues when installing //A//.
* //S// already has a number of packages installed, but they may not be the correct versions. Unfortunately, some of those packages, in the installed versions, may be needed by //S//, so we can't just clobber the installed packages with our versions. Somehow we need to install our own versions in parallel with the existing ones.
* We may want to have several versions of //A// installed at the same time, and these may have differing requirements. The most common case where we might want to do this is upgrading //A// - since the upgrade process may take some time, we want to leave the old version of //A// up and running for at least the initial part of the upgrade. So we not only may need to have versions of packages incompatible with //S//, but we may need versions incompatible with other installations of //A// as well. We would like to do this in an efficient way - if two versions of //A// need the same version of a package //P__1__// we would like to be able to share the storage for that package between the installations.
* Once we've defined how to deploy //A// onto //S//, we want the deployment to be automatic - scriptable in other words - and quick. It should also know when something has gone wrong, so scripts can detect failures.
* As far as possible skel should be neutral about the kind of application it's trying to deploy. In particular a language-specific deployment tool is no good - being able to deploy pure Perl applications is no use if //A// is in Python using Java and C++ libraries.
* As far as possible skel should disturb the platform as little as possible during a deployment. Even if we could get away with upgrading various parts of the platform, we'd like to avoid doing so if we can. Obviously this is not always possible - for instance if we require certain kernel features such as support for a given device, we have to have the correct kernel version.
* Once //A// is installed, we need to be able to run it. In particular we may need to tell it where to find its various supporting packages, as well as possibly providing configuration information. Almost by definition, some of these will not be in their default locations (because they would then clash with the platform's versions). We want to do this in the simplest way possible, and in particular we don't want applications to have to be written with a particular deployment framework in mind if we can avoid it. We also don't want to wire in locations at build time, which is just stupid.
[This section describes the state of things in mid 2004.]
~PiSearch is a prototype search application written in Python. Unfortunately it is far from being pure Python, and even if it was it would still have deployment issues.
The platform on which we want to deploy ~PiSearch is the standard lastminute.com Linux production system. This is (I think) a RH 7.3 system with a more recent kernel. (Linux platform versioning is insanely difficult).
~PiSearch is written assuming a Python version of at least 2.3. (Since Python is essentially a single-implementation language, it's meaningful to confuse the language version with the implementation version). Unfortunately the platform has Python 1.5.2 which is radically older. There is no chance that ~PiSearch will run on this Python version. There is also a high chance that if we globally upgraded the platform's Python, this would break other things.
~PiSearch uses libraries written in C, C++ and Python, as well as some Java code. Very few if any of these libraries exist on the target platform. ~PiSearch itself has some C code - both as glue to external libraries and for (probably misguided) optimisation purposes.
This is not the end. C++ and C code relies on a fair amount of `standard' runtime support in the shape of shared libraries. For C these libraries are fairly stable (although they are far from completely stable on Linux, witness the great libc 5 / libc 6 catastrophe). For C++ the runtime is very much more brittle - in the gcc 2 era almost every version of gcc had a subtly incompatible C++ standard library. The C++ standard libraries that ship with gcc 3 are better, but none of them are compatible with any of the gcc 2 versions. Further, the C++ language compiled by gcc has changed between gcc 2 and 3 (and probably between almost every version of gcc 2). Finally, ~RedHat, in their infinite wisdom, shipped an obscure and incompatible version of gcc with RH 7 (gcc 2.96, which was never an official GNU version of gcc, and is subtly incompatible with 2.95 (the last 2.x GNU release) and very incompatible with 3.x).
So, in fact, we end up needing the C/C++ runtime libraries as well. And we definitely can't change the deployment platform's libraries.
Finally, once we have somehow got all of this stuff installed on the machine, we need to tell ~PiSearch where the various supporting libraries are, and tell them where their runtime support is. For the C/C++ shared libraries we know for certain that they will not be in the standard locations.
This is just as nasty as it looks.
!!!Using the platform's package manager
One obvious strategy would be to use the standard package manager supported by the platform, or possibly some non-native package manager. Platform package managers, such as RPM or APT can deal with installing and uninstalling sets of files (`packages'), managing dependencies between packages, and much more.
Unfortunately, platform package managers have some problems.
* Their purpose in life is generally to have one version of a package installed on a system. They don't have support for maintaining multiple versions of a package (presumably in different locations).
* Packaging an application can be fairly hard work, and may not agree with the deployment framework naturally supported for the application - distutils for Python, for instance.
* They are often mildly platform-specific.
It's possible to work around some of these issues, at least in part. Platform package managers generally have support for installing `client' machines - in particular they must support the installation of an OS whose system disk(s) are mounted from a `miniroot' system booted from CDROM or the network prior to the first boot of the installed OS. This support could be used to allow us to maintain our own package database, which wouldn't interfere with the platform's database. Unfortunately we then lose most of the benefit of the dependency checking done by the package manager, because many dependencies will not be satisfied (as the packages are in the wrong database). We would also need a separate database for every installation of the application, and we would then generally end up with multiple copies of packages which would eat a lot of disk space.
Using the platform's package manager could probably be made to work, but it would almost certainly be pretty clumsy. Rather than try and get this to work, skel has been designed to be as neutral as possible - skel packages could be (deployed) platform packages, so skel could be used in conjunction with the platform's package manager.
!!!Modifying or extending a currently unsuitable build/deployment system
I considered doing this - for instance modifying Ant. But it's a lot of work to do this - first you have to understand the existing system, and then you have to write all sorts of additions to it, which will need to be maintained, merged into each new release, and so on. This sounded like too much work, and an ongoing maintenance nightmare to boot.
!!!Just hacking
This is the traditional approach - just install a bunch of stuff in some directories and then muck around with environment variables and so on until things work. Be careful to ensure job security by hard-wiring paths wherever possible, and under no circumstances document anything.
Skel is largely a reaction to this kind of lossage.
!!!Using an existing deployment tool which meets our requirements
Systems like this probably exist, however they are remarkably hard to find, and they are generally not free software since they are so hard to get right. I've previously been involved in writing such a system in fact, largely because it was very hard to find anything that did what was needed. There is so much variability in this kind of deployment problem that rather few general-purpose systems exist.
Further, any general-purpose systems will have to address an enormous range of problems, and will therefore be complex. This means they will require a lot of work to master, even where they are well-designed.
Skel was designed specifically to do as little as it could get away with, to make deployment easy enough to understand. I think it's likely that implementing, deploying and configuring skel is easier (and therefore cheaper!) than simply deploying and configuring a grandiose general-purpose system.
!!!Using JumpStart / Kickstart
Solaris JumpStart is a tool for automatically configuring and deploying Solaris systems based on network-accessible databases of configuration information, packages and OS images. RedHat have a system called Kickstart which is essentially a JumpStart knockoff. Both of these systems support scripted installation and configuration of packages during the installation based on a notion of the machine's `role'.
Although a system like this doesn't meet the requirements (roughly) outlined above it could probably do what we need. If we could reinstall a system unattended from cold in 15 minutes, and centrally define its role, including versions of packages and so on, then we could simply avoid many of the issues with compatibility between applications and upgrades and so on, since we'd never upgrade an existing application, we'd just reinstall with a new version.
And of course we could then have some hairy management software which would automatically deploy large numbers of machines, dealing with redeployment and so on if machines failed, or on administrative request. Something like [[this|missing]] in fact.
Skel was written with this sort of environment in mind - it could be run during the postinstall phase of something like JumpStart to do the final deployment of an application. However doing something like this right is obviously a fair amount of up-front work.
!!!Documented procedures
A penultimate alternative would be to not automate anything, but to simply document procedures very carefully, and follow the procedures manually. This is not really a very good choice, but it's better than just hacking.
Finally, we come to skel.
/***
|''Name:''|LegacyStrikeThroughPlugin|
|''Description:''|Support for legacy (pre 2.1) strike through formatting|
|''Version:''|1.0.2|
|''Date:''|Jul 21, 2006|
|''Source:''|http://www.tiddlywiki.com/#LegacyStrikeThroughPlugin|
|''Author:''|MartinBudden (mjbudden (at) gmail (dot) com)|
|''License:''|[[BSD open source license]]|
|''CoreVersion:''|2.1.0|
***/
//{{{
// Ensure that the LegacyStrikeThrough Plugin is only installed once.
if(!version.extensions.LegacyStrikeThroughPlugin) {
version.extensions.LegacyStrikeThroughPlugin = {installed:true};
config.formatters.push(
{
name: "legacyStrikeByChar",
match: "==",
termRegExp: /(==)/mg,
element: "strike",
handler: config.formatterHelpers.createElementAndWikify
});
} //# end of "install only once"
//}}}
Skel and this manual are copyright © 2004 lastminute.com, 2006 Tim Bradshaw.
All rights reserved.
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
* Neither the names of the copyright holders nor their contributers may be used to endorse or promote products derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
[[Reading this manual|Reading this manual]]
[[License]]
----
<<tagCloud excludeLists excludeSearch systemConfig>>
* [[Problems, and some solutions / 3 Non-standard package layouts]]: I can't work out how to have an {{{ol}}} with significant chunks of verbatim text in.
/***
!Document version
{{{$Id: //depot/www-tfeb-org/main/www-tfeb-org/html/programs/skel/Skel.html#2 $}}}
!~TiddlyWiki versions
|noBorder topAlign|k
|<<version>> |current |
|2.3.0|20071219 |
|2.2.5|20070920 |
|2.1.3 |20061213 |
|2.1.0 |20061002 |
|2.0.11 |20060629 | start |
!Shadows
<<list shadowed>>
!System configuration
***/
/*{{{*/
config.options.chkHttpReadOnly = true;
config.options.txtUserName = "Tim Bradshaw";
/*}}}*/
!!On skel
<<slider Skel/Notes/OS [[Notes / 1 / On skel]] / "problems with skel">>
!!On this manual
<<slider Skel/Notes/OTM [[Notes / 2 / On this manual]] / "many and various issues">>
!!On applications
<<slider Skel/Notes/OA [[Notes / 3 / On applications]] / "many are broken">>
!!Glossary
<<slider Skel/Notes/Glossary [[Notes / 4 / Glossary]] / "words">>
* Symbolic vs hard links. currently all the links created by skel are symbolic. Some of these - those created by stow - can't be changed, but for others it's not clear if symbolic links or hard links would be better. Hard links would be somewhat faster, but also somewhat more obscure. Perhaps this should be configurable.
* Configuration file syntax. The config file reader is at least common between all the programs, but it should be made to use a proper lexical analyser rather than the current ad-hoc mass of regexps, which leads to weirdness like whitespace always being magic. There may be a standard Perl module for this which I should use.
* Generalised configuration sources. There should be some more general source of configuration information - it would be particularly nice to get environment settings from some centralised source for instance, such as YP.
* Configuration coherence. The various commands have varying ways of geting configuration information: {{{cpskel}}} has rather few parameters, and uses some (undocumented) mechansisms to default things from the environment; {{{cfskel}}} suffers fairly severely from `second-system' effect (albeit in miniature), and goes to the opposite extreme and parameterises almost everything, based on the configuration file reading code that was needed anyway. The wrapper uses yet another environment mechanism. These should be integrated somehow, or the different mechansisms should at least be documented and justified.
* Non-standard package locations. It would be nice if there was a way to persuade {{{cfskel}}} to install packages in some non-standard location and then stow them there. The aim would be to make installation of packages which clash easier. I have no idea what the interface to this would be like, though.
* Missing sections. These should be written.
* Better documentation. Several things are only partially described and should have more written about them.
* Manual coherence. It needs to be (first/third person active/passive, capitalisation &c).
* [[TiddlyWiki|http://www.tiddlywiki.com/]] conversion. Much work is needed.
----
See [[Structural notes]] about the somewhat experimental conversion to [[TiddlyWiki|http://www.tiddlywiki.com]] from the original [[TWiki|http://twiki.org/]].
* Environment access. Most languages which have Unix implementations support easy access to the environment. A possible exception is Java. {{{java.System.getenv()}}} is deprecated in 1.4.2 (and probably earlier versions). Fortunately, Sun have seen the light, and in 1.5 it will no longer be deprecated, since it is no less portable than command line arguments, say. See [[here|http://intgat.tigress.co.uk/rmy/java/getenv/case.html]] for a decent summary of the situation.
* Unix. Any sufficiently Unix-like system, including Linux. Cygwin may be sufficiently Unix-like in this sense, although I'm not sure if its implementation of symbolic links is good enough.
* Platform. The system onto which we're trying to deploy something. The combination of hardware, OS and other software and configuration which we can assume is there before we start. One aim of skel is to perturb this platform as little as possible when deploying something.
!!Package clashes
<<slider Skel/PAS/1
[[Problems, and some solutions / 1 / Package clashes]]
/ "packages with clashing files or structures">>
!!Non-standard `bin' directories
<<slider Skel/PAS/2
[[Problems, and some solutions / 2 / Non-standard `bin' directories]]
/ "packages with executables in odd places">>
!!Non-standard package layouts
<<slider Skel/PAS/3
[[Problems, and some solutions / 3 Non-standard package layouts]]
/ "packages which don't look anything like skel wants them to">>
When stow makes the links for packages, it checks for conflicts between packages: each link it creates to a file must belong to exactly one package. This is simply a sanity requirement - if there was a situation like:
* //package-1//
** {{{README}}}
* //package-2//
** {{{README}}}
then how would stow know which {{{README}}} to install?
Unfortunately some bits of software which could otherwise be used as packages for skel come with files with names like {{{README}}}, {{{INSTALL}}} and so forth at the top level and these almost inevitably cause conflicts. Sun's JDK is an example of a package like this, and the Apache Jakarta ~JMeter binary distribution clashes with it for rather spurious reasons.
Packages can clash for real reasons as well. For instance, if two packages have an executable called {{{bin/java}}} then there is a genuine problem which has to be addresed - we have to choose one of them.
There are two solutions to this problem. Both of them are really just workarounds, since the problem is real.
''1.'' Modify the packages so they don't clash. For instance if two packages have a {{{README}}} at the top level, delete one of them. This solves the problem, but it requires work, and in the cases where the clashes are more serious than this it can cause compatibility issues. It's particularly annoying for third-party packages as it can make automating the installation process harder: what used to be
# {{{zcat package.tar.gz | tar xfp -}}}
becomes
# {{{zcat package.tar.gz | tar xfp -}}}
# {{{(cd package; rm README LICENSE bin/README)}}}
I'm not going to talk more about this solution.
''2.'' Install the packages in different locations and then fiddle with the environment so they can both be found. This is ugly, but since skel can fiddle with the environment in almost arbitrary ways, it's not as bad as it looks. For concreteness, consider these two third-party packages:
* Sun's ~J2SE RC of 1 September 2004, available from http://java.sun.com/j2se/1.5.0/, unpacked as {{{jdk-1.5.0}}};
* The Apache Jakarta ~JMeter, from http://jakarta.apache.org/jmeter/, version 2.0.1, unpacked as {{{jakarta-jmeter-2.0.1}}}.
These packages clash because of several commonly-named files at the top level - {{{README}}} and so on. The solution to this is to
# cause skel to install one of them in a non-standard location;
# tell skel that it needs to wrap the executables in the non-standard `bin' directory;
# configure the environment for wrapped programs so that the non-standard `bin' directory is in {{{PATH}}}.
''Installing packages in non-standard locations.'' There's [[no really good way|missing]] of telling skel to install a package in a non-standard place. The easiest thing to do is to put clashing packages below an {{{opt/}}} directory (it can have any name, but {{{/opt/}}} is a common Unix directory for third-party software). In this case we'll move ~JMeter. What we do is create a `fake' package in the repository called {{{opt-packages}}} which has a single directory, opt/, which has the real package or packages below it. The structure we want to end up with is something like this:
[img[/opt package structure|figures/opt-package-structure.png]]
It's unfortunate that you need so many levels of directory hierarchy to do this.
''Telling cpskel to use the opt-packages.'' Simply refer to them in the packages file:
{{{
# all the opt packages
opt-packages
}}}
Note that this will make links to all the packages in {{{opt-packages}}}. Fortunately, stow should optimise the number of links, and there should be no clashes since each package is now in a different directory.
''Wrapping non-standard `bin' dirs.'' See [[below|Problems, and some solutions / 2 / Non-standard `bin' directories]].
''Configuring the environment.'' Finally, the environment needs to be configured to point to the extra `bin' dirs (and possibly extra library directories, although not in this case):
{{{
# JMeter version
set JMETER_VERSION 2.0.1
# for JMeter
prepend-maybe PATH ${ROOT}/opt/jakarta-jmeter-${JMETER_VERSION}/bin
}}}
Once all this is done, then ~JMeter should be usable without package clashes. (In this case, it turned out that ~JMeter doesn't work with the new JDK yet, so all this effort was wasted, but it is applicable in other cases too).
There are at least two reasons why it may be necessary to have `bin' directoriess in non-standard places:
* when dealing with clashing packages as [[above|Problems, and some solutions / 1 / Package clashes]], each package will have a `bin' directory which is in a non-standard place, and all these directories need to be wrapped;
* even non-clashing packages may have additional `bin' directories, such as {{{sbin/}}} for instance.
Skel deals with both these cases by allowing you to specify the location of other `bin' directories to wrap.
{{{cfskel}}} has several parameters which tell it where the `bin' directories are. These can be given either on the command line, or by setting parameters in the configuration file. The easiest way to do this, I think, is to set the variables in the configuration file. Here's an excerpt from {{{package-configuration}}} which does this:
{{{
# set extra binary directories: jakarta JMeter
extra-bin-dirs opt/jakarta-jmeter-2.0.1/bin
}}}
(Several extra `bin' directories can be set by giving {{{extra-bin-dirs}}} as a colon-separated list in the usual Unix way.)
''Note'' it is OK to wrap {{{sbin/}}}, even though skel's own commands live in this directory: they're constructed in such a way that cfskel will not attempt to re-wrap them (including itself!).
Skel really wants all its packages to have the same basic layout - in particular it likes them all to have a `bin' dir in a standard location relative to the package root (typically {{{bin/}}}), not to have any files in the top-level package directory, and so on.
Not all packages are like this. For instance, Eclipse has its executable in the top level of the package. This doesn't fit well with skel's system.
An easy solution for packages like this is to install the package below {{{opt/}}} as [[above|Problems, and some solutions / 1 / Package clashes]], and then to write a tiny `wrapper' package which has a single executable script which simply arranges to run the appropriate real executable. So, in this case what we do is:
1. Install the package under the {{{opt-packages}}} `fake' package, say in {{{.../opt-packages/opt/eclipse-3.1m1/}}}.
2. Create a tiny {{{eclipse-wrapper}}} package, whose binary directory contains a script {{{eclipse}}} which is something like this:
{{{
#!/bin/sh -
#
# Wrapper to run eclipse for skel
#
E=${ROOT}/opt/${ECLIPSE}/eclipse
if [ ! -x "${E}" ]
then
echo "Can't find eclipse executable, tried ${E}" 1>&2
exit 1
else
exec "${E}" "$@"
fi
}}}
3. Name the {{{eclipse-wrapper}}} package in the package file, as well as all the opt packages:
{{{
# Get all the opt-packages (includes jdk and eclipse)
opt-packages
# The eclipse wrapper
eclipse-wrapper
# and stow of course
stow-1.3.3
}}}
4. Set up the environment with a line like:
{{{
# eclipse
set ECLIPSE eclipse-sdk-3.1m1
}}}
to tell the script where eclipse is.
And that's it.
The manual is structured as a folded outline. Selecting the {{{/}}} characters will fold or unfold a section. You can also search or look for tags. Everything works the way you'd expect in a [[TiddlyWiki|http://www.tiddlywiki.com]]: in particular it may not work in some older browsers.
The little glyphs that float to the right of the page allow you to do things like hide the right hand column to give you more screen space for text, jump between open tiddlers, and so on (it's a [[hover menu|HoverMenuPlugin]]). Hiding the right column is very useful if you want to read the manual in a fairly tall, narrow window (so you can fit a terminal window next to it, say).
Most of the left hand column is a `tag cloud' which shows the various tags on tiddlers, with bigger ones occurring more often.
[[About skel and this manual]].
a tool for deploying hostile applications on Unix
!Introduction
<<slider Skel/intro [[Introduction]]
/ "what skel is trying to achieve, and trying to avoid">>
!Skel's world
<<slider Skel/SW [[Skel's world]]
/ "the things that are">>
!The skel user's guide
<<slider Skel/UG [[The skel user's guide]]
/ "reference manual">>
!Problems, and some solutions
<<slider Skel/PAS [[Problems, and some solutions]] / "obscurities and workarounds">>
!Wrapping: the terrible truth
<<slider Skel/Wrapping [[Wrapping: the terrible truth]] / "it's worse than you think">>
!About skel and this manual
<<slider Skel/About [[About skel and this manual]] / "a history of infamy">>
!Notes
<<slider Skel/Notes [[Notes]] / "useful and useless things">>
!Bugs
<<slider Skel/Bugs [[Bugs]] / "broken things">>
<<tiddler [[Skel's world / 1 / prelims]]>>
!!Skeletons
<<slider Skel/SW/2 [[Skel's world / 2 / Skeletons]]
/ "what skeletons are">>
!!Configuration files
<<slider Skel/SW/3 [[Skel's world / 3 / Configuration files]]
/ "configuration files for skel">>
!!{{{cpskel}}}
<<slider Skel/SW/4 [[Skel's world / 4 / cpskel]]
/ "the cpskel command">>
!!Roots
<<slider Skel/SW/5 [[Skel's world / 5 / Roots]]
/ "the tops (or bottoms) of trees">>
!!Packages
<<slider Skel/SW/6 [[Skel's world / 6 / Packages]]
/ "what a package is">>
!!Package repository
<<slider Skel/SW/7 [[Skel's world / 7 / Package repository]]
/ "where packages live">>
!!{{{cfskel}}}
<<slider Skel/SW/8 [[Skel's world / 8 / cfskel]]
/ "the cfskel command">>
!!Stowing
<<slider Skel/SW/9 [[Skel's world / 9 / Stowing]]
/ "making links">>
!!Wrapping
<<slider Skel/SW/A [[Skel's world / A / Wrapping]]
/ "controlling the environment">>
!!Directory structures
<<slider Skel/SW/B [[Skel's world / B / Directory structures]]
/ "directory structures created by skel">>
!!Symbolic links
<<slider Skel/SW/C [[Skel's world / C / Symbolic links]]
/ "links created by skel">>
Skel was designed to support deploying [[PiSearch|Introduction / 2 / A specific deployment example: PiSearch]], although with luck it should be useful for other applications which have deployment requirements as above.
Skel is intended to be about as simple as it is possible to be, while being usable and automatic. For instance:
* skel does not do dependency checking - instead you simply tell skel the list of packages required which it will then install;
* skel does not do build management;
* skel does not do configuration management and does not have specific interfaces to configuration management systems;
* skel makes fairly strong demands on the packages which it installs, rather than bending over backwards to support fractious packages;
* skel is not extensible by scripts - rather it is usable from scripts.
The reason skel is like this is that I wanted to make it work fast, and I've previously worked on much hairier and more comprehensive deployment systems most of whose features were never used in practice.
!!The things in skel's world
[img[Skel's world|figures/skels-world.png]]
The initial starting point for a deployment using skel is a skeleton. This is simply a directory containing skel itself and some default configuration files: it contains just the bare bones needed to bootstrap the deployment of an application.
Except in unusual circumstances, there should be no reason to modify the initial skeleton - configuration files and packages are installed into it after copying, so any application- or deployment-specific data can come from these.
A skeleton contains some subdirectories which contain skel:
* {{{sbin}}} contains the skel commands;
* {{{etc}}} contains the default configuration files;
* {{{lib/skel/site_perl}}} contains Perl modules used by skel;
* {{{bin}}} contains a utility useful after deployment.
Skel is driven by some configuration files. These describe which packages are needed and various other things. They are all in a similar, simple format.
A skeleton contains some initial default configuration files, but during the process of copying the skeleton other configuration files would normally be installed into the root being created. These end up in the {{{etc}}} directory of the initial root.
{{{cpskel}}} (`copy skeleton') is the tool which takes a skeleton and some optional configuration files and created an initial root. cpskel itself lives in {{{.../sbin/cpskel}}}.
Skel installs applications under `roots', which are directories created by {{{cpskel}}} from skeletons and configuration files, and then fleshed out with packages and further configured. A root can be anywhere in the Unix filesystem, and should not exist before {{{cpskel}}} is run.
A root will typically look like a subset of a `standard' Unix filesystem - it will have {{{bin}}}, {{{etc}}}, {{{lib}}} and {{{sbin}}} subdirectories as well, possibly as others. Skel itself doesn't enforce any particular convention for roots, other than that, because they are created from skeletons, they will contain the initial skeleton directories.
Skel assumes the existence of a number of `packages' in some central location in the filesystem. These packages are not RPM packages or anything like that: they're just directories containing files. Skel doesn't care at all about what is in these packages, or how they came into existence. Conventionally, packages are named as //thing//-//version//, and skel has some mild support for this - if you specify, say {{{foo-2}}} then skel will look for {{{foo-2*}}} and choose the most recent version.
Skel's notion of `package' is designed to be compatible with the fairly standard `prefix-installation' method of installing software supported by the GNU autotools and most other package configuration systems. For instance to install gcc 3.3.4 from a source tarball, {{{gcc-3.3.4.tar.bz2}}}, as a skel-compatible package, under a package directory {{{/local/packages/}}} you would do this:
{{{
$ bzcat gcc-3.3.4.bar.bz2 | tar xfp -
$ mkdir gcc-build; cd gcc-build
$ ../gcc-3.3.4/configure --prefix=/local/packages/gcc-3.3.4
$ make bootstrap
$ make install
}}}
However Skel packages could be the result of RPM installations or any other mechanism of getting software on to the machine.
An important property of a skel package is that it should be relocatable - it shouldn't care where in the filesystem it ends up. So for instance if gcc is built with a prefix of {{{/local/packages/gcc-3.3.4}}}, then it should be possible to move it to, say, {{{/local}}} and have it still work. It may be necessary to configure the environment to relocate a package - for instance setting {{{PATH}}}, {{{LD_LIBRARY_PATH}}} or other variables. Skel can arrange for this to happen when the application is run. The important thing is that packages should not wire in absolute pathnames for parts of themselves. Fortunately most widely-distributed packages don't do this any more.
Skel could make use of non-relocatable packages, but these would only work under a root at a particular, known location, which would largely defeat the point of using skel.
Skel does not care what the contents of a package are, although they will normally look like standard Unix filesystems, with {{{bin}}}, {{{lib}}} &c directories. This is the standard structure produced by an autotools prefix-installation also.
In fact, this is a slight lie: skel does need to know know the directory which contains applications, so it can wrap them. This directory is defaultly {{{bin}}}.
There are some packages which are always required by skel (at present only one).
Skel does not enforce any naming convention for packages, although it has some limited support for versioned package names.
The package repository is simply the directory where all the packages live. This directory needs to be locally available on the plaform where skel is doing deployments. The repository could be mounted from an NFS file server, although the packages in the repository afe used during the running of the application (they are linked to, not copied), so it may be better to make a copy of the repository on local storage.
The process of fleshing out and otherwise configuring an initial root directory is controlled by {{{cfskel}}}, which lives in {{{sbin/cfskel}}} in the skeleton and initial root. {{{cfskel}}} reads configuration files installed by {{{cpskel}}}.
{{{cfskel}}} installs packages into the initial root using [[GNU stow|http://www.gnu.org/software/stow/]] which is a tool which will create symbolic link farms to support this kind of package installation. Stow produces symbolic link farms with the minimal number of links, and will also detect any clashes between packages - for instance if two packages both contain the same file stow will detect this and fail, causing {{{cfskel}}} to fail in turn.
After stowing, the root directory contains a large number of symbolic links pointing to the packages required. This means that multiple roots can be created and can share the same set of packages. The links are relative by default but can be forced to be absolute.
Stow itself needs to be found by {{{cfskel}}}. It doesn't assume that stow is preinstalled on the system, but rather looks for a suitable package contraining stow, and then runs stow from that package to stow itself, and then all the other packages. This package is always required by skel. All fleshed out roots contain stow because of this.
This is nearly the end. Unfortunately the last stage is also the most obscure.
We want to be able to provide various sorts of information to the application and any supporting packages:
* applications may want to know where the root is, so they can find data ad so on;
* paths may need to be set, in particular {{{PATH}}} and {{{LD_LIBRARY_PATH}}};
* packages may need other kinds of location information;
* default values of options and parameters may be needed.
Skel does all this by means of the Unix environment. In fact, the only way that skel influences the running of applications is by the environment. This means that applications deployed by skel don't have to worry about strange arguments or anything like that - they just get the arguments that the user provided. However it means that if applications want to listen to parameters that skel could set, they can only do this via the environment. [[Most languages|missing]] provide fairly easy access to the Unix environment.
In order to control the environment seen by applications, Skel needs to get control before any application runs. it does this by `wrapping' all executable files that it finds in the application directory (defaultly {{{bin}}}). Wrapping an application consists of renaming it to a `private' name, and then replacing it with a symbolic link to a small wrapper script. This script:
# sets some initial environment variables;
# reads some optional configuration files which specify further variables to be set, cleared, checked, appended to and so on;
# execs the original wrapped executable in the augmented environment.
This allows relatively complete control over the environment seen by applications.
The configuration files which control the environment can be installed by {{{cfskel}}}, and several configuration files can be read - a global one and a per-application one. These files contain a simple language which allows you to control the environment in fairly restricted ways. For example
{{{
set JAVA_HOME ${ROOT}/usr/java
prepend CLASSPATH ${ROOT}/lib/java
}}}
will set {{{JAVA_HOME}}} and prepend an entry to {{{CLASSPATH}}}. {{{ROOT}}} is preset by the wrapper to be the full pathname of the root.
Wrapping applications like this is a fairly common trick. Skel has to carry it one stage further though: skel itself (and hence the wrapper) is written in Perl, and uses some Perl modules which are part of skel. We don't want to install these in the normal Perl module directories, but keep them within the skeleton. So the skel scripts themselves are wrapped with a tiny shell script wrapper, which finds the perl modules and invokes Perl with suitable options. This means that the final structure of the application directory is reasonably complex with symbolic links pointing to the shell wrapper, the perl wrapper and obscurely-named wrapped files. {{{cfskel}}} manages all this hairiness. See [[below|missing]] for the full story on wrapping.
Once everything in the root is wrapped, skel's job is done: the application should be runnable. Sharp-eyed readers will have noticed that I haven't actually mentioned an application in this section: that's because, from skel's point of view, there isn't one. An application is just another package which skel stows into the root, so it has exactly the same structure as any other package.
The directory structures produced by skel are fairly complex, and contain a lot of symbolic links. The following diagrams are both incomplete - some structure is omitted or glossed over - and incorrect - some of the more fiddly link structures are glossed over.
!!!Skeleton and initial root
[img[Skeleton and initial root|figures/skeleton-and-initial-root.png]]
The skeleton structure and the initial root are essentially the same. The differences are:
* {{{cpskel}}} only copies the bits of the skeleton it's interested in;
* {{{cpskel}}} may copy in configuration files from elsewhere if asked (normally it would do this as the skeleton's configuration files are unlikely to be correct);
* {{{cpskel}}} may also may create one or more data directories (not shown in the diagram).
The actual structure of the {{{sbin}}} directory is more complex than shown here, as even the skel scripts are wrapped by a small shell wrapper.
!!!Fleshed out root, simplified
[img[Fleshed out root, simplified|figures/fleshed-out-root-simplified.png]]
This is a simplified and partial picture of a root after packages have been stowed. The two packages here are {{{gcc-3.3.2}}} and {{{xenia-0.0}}}. The packages `live' under {{{stow/}}} and symbolic links are created by stow from the `logical' location to the actual location. As you can see, each package is essentially a little file tree in its own right, with its own binary and library directories. The root is constructed as a `link farm' or `shadow tree' from all the packages, with symbolic links which point into the package directories.
This diagram is simplified, and incorrect in several ways:
* stow actually produces a highly optimised link structure, constructing only the links that it needs to - if a directory only occur in one package, then the symbolic links will be to the directory, not to the individual files within it;
* the executables in {{{bin/}}} will be wrapped, which is not indicated at all;
* in fact, the packages don't live under {{{stow/}}} at all - see the next diagram.
!!!Fleshed out root, details of package links
[img[Fleshed out root, details of package links|figures/fleshed-out-root-detail.png]]
This diagram shows the package structure in more detail, and more correctly. The packages do not actually live under {{{stow/}}}, but in the package repository. Symbolic links are created under {{{stow/}}} which point into the package repository, and the `logical' links are now two stages removed from the real location. So, for instance, when trying to run {{{bin/gcc}}} the following happens (assuming relative links, see below):
# look for {{{bin/gcc}}}: it is a symbolic link to {{{../stow/gcc-3.3.4/bin/gcc}}};
# start following link, reading {{{../stow/gcc-3.3.4}}} ...
# ... which is a symbolic link, pointing to (say) {{{../../../packages/gcc-3.3.4}}};
# we're now looking for {{{../../../packages/gcc-3.3.4/bin/gcc}}} ...
# ... and this will succeed, all being well.
In fact, since {{{bin/gcc}}} is a binary, it is wrapped, and things are even more complex than this.
The end result of this is that there is only a single copy of a package, which lives in the package repository, with possibly multiple roots pointing at it. It's also easy for multiple different versions of a package to exist in the repository, with different roots pointing at different versions. There can be several package repositories, but a single root can't point at more than one (at present).
Skel makes heavy use of symbolic links to construct roots while sharing package data, and also to wrap executables. There are some issues with symbolic links which are worth discussing.
''Performance.'' Symbolic links are often fairly expensive to look up, especially over NFS. Multiple levels of links are especially bad. Fortunately, I think that modern systems and networks are fast enough that this will not significantly hurt performance. For long-running server daemon applications the rate of lookups for executables should be fairly low, at least.
''Relative or absolute.'' Symbolic links can point to relative or absolute filenames. Which of these is chosen affects the relocatability of a root - relative symlink paths within the root mean that if it's moved then it will continue to work. However relative paths to the package repository mean that if the root is moved it won't work unless the package repository is moved in an equivalent way.
Currently skel uses relative links for both links within a root and from the root to the package repository by default, with an option to cause it to use absolute links for both. This is not entirely satisfactory: by default there should be relative links within the root and absolute from root to package repository, with options to override either individually. This will be implemented in a future version.
[[Possibly skel should use hard links sometimes.|missing]]
/***
|''Name:''|SparklinePlugin|
|''Description:''|Sparklines macro|
***/
//{{{
if(!version.extensions.SparklinePlugin) {
version.extensions.SparklinePlugin = {installed:true};
//--
//-- Sparklines
//--
config.macros.sparkline = {};
config.macros.sparkline.handler = function(place,macroName,params)
{
var data = [];
var min = 0;
var max = 0;
var v;
for(var t=0; t<params.length; t++) {
v = parseInt(params[t]);
if(v < min)
min = v;
if(v > max)
max = v;
data.push(v);
}
if(data.length < 1)
return;
var box = createTiddlyElement(place,"span",null,"sparkline",String.fromCharCode(160));
box.title = data.join(",");
var w = box.offsetWidth;
var h = box.offsetHeight;
box.style.paddingRight = (data.length * 2 - w) + "px";
box.style.position = "relative";
for(var d=0; d<data.length; d++) {
var tick = document.createElement("img");
tick.border = 0;
tick.className = "sparktick";
tick.style.position = "absolute";
tick.src = "data:image/gif,GIF89a%01%00%01%00%91%FF%00%FF%FF%FF%00%00%00%C0%C0%C0%00%00%00!%F9%04%01%00%00%02%00%2C%00%00%00%00%01%00%01%00%40%02%02T%01%00%3B";
tick.style.left = d*2 + "px";
tick.style.width = "2px";
v = Math.floor(((data[d] - min)/(max-min)) * h);
tick.style.top = (h-v) + "px";
tick.style.height = v + "px";
box.appendChild(tick);
}
};
}
//}}}
Chapters have 1st-level headings (outside their text, in [[Skel]]). Sections within chapters are named chap / n / section where n is [1-9A-Z], and start with an implicit 2nd-level heading. subsections within a section have 3rd-level headings. In theory they could be named as chap / n / sec / m / subsec, but they're all included in their parents. There should be no 4th-level headings.
There is confusion about when there should and should not be third-level headings.
In diagrams dirs are tungsten, files are lead, descriptions are lead. dir arcs are aluminium. Shadows are silver (arcs and nodes). symlinks (arcs and nodes) are blueberry, and arcs have plain arrowheads.
----
<<tagging meta>>
/***
Suppress hover menu in printed output
***/
/*{{{*/
@media print {
#hoverMenu {display: none ! important;}
}
/*}}}*/
/***
!Mucking around with tables
***/
/*{{{*/
/* Table with no overall border */
table.noBorder {
border-style: hidden !important;
}
/* Table with no borders at all */
table.noBorders *, table.noborders {
border-style: hidden !important;
}
/* Table with rows top aligned */
table.topAlign tr {
vertical-align: top ! important;
}
/*}}}*/
/***
!Overrides for TaskMacroPlugin
***/
/*{{{*/
table.task.done td.status,table.task.done td.description {
/* default is #ccc, which is too pale */
color: #a0a0a0;
}
/*}}}*/
/***
''Plugin:'' Tag Cloud Macro
''Author:'' Clint Checketts
''Source URL: http://checkettsweb.com/styles/themes.htm''
!Usage
<<tagCloud>>
!Code
***/
//{{{
version.extensions.tagCloud = {major: 1, minor: 0 , revision: 0, date: new Date(2006,2,04)};
//Created by Clint Checketts, contributions by Jonny Leroy and Eric Shulman
config.macros.tagCloud = {
noTags: "No tag cloud created because there are no tags.",
tooltip: "%1 tiddlers tagged with '%0'"
};
config.macros.tagCloud.handler = function(place,macroName,params) {
var tagCloudWrapper = createTiddlyElement(place,"div",null,"tagCloud",null);
var tags = store.getTags();
for (var t=0; t<tags.length; t++) {
for (var p=0;p<params.length; p++) if (tags[t][0] == params[p]) tags[t][0] = "";
}
if(tags.length == 0)
createTiddlyElement(tagCloudWrapper,"span",null,null,this.noTags);
//Findout the maximum number of tags
var mostTags = 0;
for (var t=0; t<tags.length; t++) if (tags[t][0].length > 0){
if (tags[t][1] > mostTags) mostTags = tags[t][1];
}
//divide the mostTags into 4 segments for the 4 different tagCloud sizes
var tagSegment = mostTags / 4;
for (var t=0; t<tags.length; t++) if (tags[t][0].length > 0){
var tagCloudElement = createTiddlyElement(tagCloudWrapper,"span",null,null,null);
tagCloudWrapper.appendChild(document.createTextNode(" "));
var theTag = createTiddlyButton(tagCloudElement,tags[t][0],this.tooltip.format(tags[t]),onClickTag,"tagCloudtag tagCloud" + (Math.round(tags[t][1]/tagSegment)+1));
theTag.setAttribute("tag",tags[t][0]);
}
};
setStylesheet(".tagCloud span{height: 1.8em;margin: 3px;}.tagCloud1{font-size: 1.2em;}.tagCloud2{font-size: 1.4em;}.tagCloud3{font-size: 1.6em;}.tagCloud4{font-size: 1.8em;}.tagCloud5{font-size: 1.8em;font-weight: bold;}","tagCloudsStyles");
//}}}
This guide is not complete: in particular the descriptions of the skel commands are not comprehensive, and a fair number of options and other details are omitted. There should be enough information to get by with, but it needs to be fleshed out.
All the skel commands, except {{{wrap}}}, live in the {{{sbin/}}} directory of the initial skeleton, and hence of the root.
!!Creating packages for skel
<<slider Skel/UG/1
[[The skel user's guide / 1 / Creating packges for skel]]
/ "how to create packages for skel">>
!!{{{cpskel}}}: copy an initial skeleton
<<slider Skel/UG/2
[[The skel user's guide / 2 / cpskel]]
/ "the cpskel command">>
!!{{{cfskel}}}: configure a root
<<slider Skel/UG/3 [[The skel user's guide / 3 / cfskel]]
/ "the cfskelc ommand">>
!!Environment control files
<<slider Skel/UG/4 [[The skel user's guide / 4 / Environment control files]]
/ "controlling the environment for commands">>
!!Controlling the wrapper
<<slider Skel/UG/5 [[The skel user's guide / 5 / Controlling the wrapper]]
/ "environment variables which can control the wrapper">>
!!{{{wrap}}}: user wrapper to run programs with the environment set up
<<slider Skel/UG/6 [[The skel user's guide / 6 / wrap]]
/ "user wrapper to run programs with the environment set up">>
Skel needs a repository of packages, including the application to be deployed. It's important to remember that a package is not anything like a tarball, an RPM file or a Debian package - it's a directory tree which will be stitched together with other directory trees to make a configured root. In order for skel to work properly, a package should be relocatable - it shouldn't care where it is (or appears to be) in the filesystem.
Many bits of software support deployment in a form suitable for use as a skel package - in fact this is why skel packages are the way they are, of course.
!!!Systems using GNU autotools
A lot of (GNU and other) systems are configured and built using the GNU autotools - [[autoconf|http://www.gnu.org/software/autoconf/]], [[automake|http://www.gnu.org/software/automake/]], [[libtool|http://www.gnu.org/software/libtool/]] and so on. These systems generally have a {{{configure}}} script which configures them and constructs makefiles which can then be used to build and install the system. Almost all systems I've seen which use the GNU autotools are relocatable, but some might not be.
To construct a package from a system using the GNU autotools, if you're sure it's relocatable you should:
# configure it with a prefix which is the package name in the repository;
# build and install it.
If you're not sure it's relocatable you can do the following:
# configure it with a prefix which is a scratch directory (in /tmp/ say);
# build and install it;
# move the scratch directory to the final location in the package repository;
# check it still works.
In either case you may need to set environment variables (using skel's wrapping facility) to help packages be relocated.
''Example: configuring and building expat, an XML parser library.'' This is for version 1.95.8, which is current at the time of writing. In this example I configure it for a temporary directory and then move the built package to the package repository, which is here {{{/home/tbradshaw/work/ps-prod/packages/}}}.
{{{
$ zcat expat-1.95.8.tar.gz | tar xfp -
$ cd expat-1.95.8
$ ./configure --prefix=/tmp/expat-19.5.8
[configure output elided]
$ make
[make output elided]
$ make install
[make output elided]
$ (cd /tmp; tar cf - expat-1.95.8) | (cd /home/tbradshaw/work/ps-prod/packages; tar xfp -)
$ rm -rf /tmp/expat-1.95.8
$ cd ..; rm -rf expat-1.95.8
}}}
Configuring directly for the package repository would be simpler, but wouldn't give you the chance to test that the package is relocatable.
!!!Python modules
The standard Python module distribution system is [[distutils|http://www.python.org/doc/2.3.4/lib/module-distutils.html]]. Python modules which use distutils generally have a {{{setup.py}}} file at their top level, which controls the build and installation.
Distutils-based modules can be installed in pretty much the same way as for GNU autoconf, except there is no separate configuration step. You simply need to do a prefix-style installation.
''Example: installing xenia, a local python module''. Xenia is an XMLRPC visualisation tool written by Richard Jones. Here, we're starting from a tarball (itself made using distutils) of xenia 0.0.
{{{
$ zcat xenia-0.0.tar.gz | tar xfp -
$ cd xenia-0.0
$ python setup.py install --prefix=~/work/ps-prod/packages/xenia-0.0
[output elided]
}}}
!!!Prebuilt systems
Many commercial applications are distributed either as a tarball, or with an installer which asks where they should live. In these cases you either need to unpack the tarball directly into the package directory, or tell the installer where to put the system.
!!!Other build systems
We probably need to mention at least Perl modules.
!!!Bootstrapping packages
Packages can have dependencies on each other at build time. For instance, to install a non-default Python we might want to use more recent gcc libraries than those shipped with the platform (for RH 7 we definitely do). So we need to build gcc, then build python in such a way that it uses the gcc we've just built. We might then want to use this Python to build a Python module
The way to do this is to construct a root with just (say) the gcc package, and then use this root to build Python, and then (perhaps) construct another root using gcc and Python packages to build Python modules. Skel's wrap command can help with this, as it can be used to get an interactive shell in a configured root. More on this [[below|missing]].
The first part of constructing a root is to copy the initial skeleton, optionally installing configuration files in the process. This is done by {{{cpskel}}}, which itself lives inside the initial skeleton.
!!!Synopsis.
{{{
cpskel [--verbose+]
[--debug]
[--config-dir config-dir]
[--config-link]
[--config-copy]
sourceroot targetroot
}}}
!!!Description.
{{{cpskel}}} copies //sourceroot// to //targetroot// optionally adding configuration files from //config-dir//, which may be linked or copied. It copies only the parts of the skeleton that it knows about rather than simply copying the entire tree. It will clao create a {{{data/}}} directory in the target root, which has {{{rwxrwxrwt}}} permissions, suitable for use as a scratch directory or for other data uses.
* {{{--verbose}}}: if given this will increase the verbosity level. This argument can be repeated (currently only twice) to make the program more verbose. The second level is probably only interesting for debugging purposes - it is really fairly verbose.
* {{{--debug}}}: turn on debugging output. This is only interesting for development of cpskel.
* {{{--config-dir dir}}}: copy or link configuration files from //dir// to the {{{etc/}}} directory of the target root. {{{cpskel}}} doesn't care what configuration files exist in //dir//, it will simply install the entire contents of the directory. Any files that would be overridden in the target directory are renamed with a {{{.dist}}} suffix.
* {{{--config-link}}}: if given, create symbolic links to the configuration files, instead of copying them.
* {{{--config-copy}}}: copy configuration files, do not link. This is the default.
* {{{--relative-links}}}: if given, configuration file links will be relative. {{{--norelative-links}}} will make them absolute. Relative links are the default.
* {{{--absolute-links}}}: the inverse of {{{--relative-links}}}.
* //sourceroot//: the skeleton to copy. This will normally be the skeleton that includes {{{cpskel}}} itself.
* //targetroot//: the root to create. This directory must not exist before {{{cpskel}}} is run.
{{{cpskel}}} also listens to a number of environment variables, which are not documented here.
In almost all cases {{{cpskel}}} will be run directly from the initial skeleton: see below.
!!!Example.
Copying an initial skeleton located at {{{/home/tbradshaw/work/ps-prod/skel}}} to {{{/home/tbradshaw/work/ps-prod/run}}}, installing config files from {{{/home/tbradshaw/work/ps-prod/etc/}}}.
{{{
$ cd /home/tbradshaw/work/ps-prod/
$ ./skel/sbin/cpskel -v --config-dir etc skel run
Source root skel/, target root run/
Making target directories
making run/sbin
making run/etc
making run/stow
making run/lib
making run/bin
Making data directories
making run/data
Syncing
syncing skel/sbin/
syncing skel/etc/
syncing skel/lib/
syncing skel/bin/
Smashing config files from /home/tbradshaw/work/ps-prod/etc/ to run/etc/
/home/tbradshaw/work/ps-prod/etc/setup.sh -> run/etc/setup.sh
/home/tbradshaw/work/ps-prod/etc/PACKAGES -> run/etc/PACKAGES
/home/tbradshaw/work/ps-prod/etc/environment -> run/etc/environment
/home/tbradshaw/work/ps-prod/etc/environment-wrap -> run/etc/environment-wrap
/home/tbradshaw/work/ps-prod/etc/package-configuration~ -> run/etc/package-configuration~
/home/tbradshaw/work/ps-prod/etc/package-configuration -> run/etc/package-configuration
/home/tbradshaw/work/ps-prod/etc/packages~ -> run/etc/packages~
/home/tbradshaw/work/ps-prod/etc/packages -> run/etc/packages
Done
}}}
!!!Bugs.
* Links should probably be absolute by default.
* It really isn't clear why {{{cpskel}}} should not copy the entire skeleton. There was once a reason, but I don't think there is now.
* The whole usage of the environment for defaulting arguments needs to be revised and documented.
!!!Prerequisites.
* cpskel needs a reasonable Unix platform.
* It needs Perl 5.6.0 or greater (it could probably be made to work with older Perls but I have not tested it, and it insists on 5.6.0.
* It needs {{{rsync}}}. This is installed by default on RH 7 and probably most other recent Unix platforms.
Once an initial root has been created, it is configured with {{{cfskel}}}. {{{cfskel}}}:
* reads a list of packages, and makes suitable links to the package repository;
* stows the packages;
* wraps all the executables so that they run in a controlled environment.
{{{cfskel}}} has a fairly large number of parameters, which it can get from command line options and a configuration file. Most of these are [[not documented here|missing]], partly because things are [[fairly incoherent|missing]] at present.
{{{cfskel}}} reads two configuration files. The locations here are the defaults, relative to the top directory of the root. Both of these files are in [[standard syntax|missing]] - the descriptions below don't mention comments or anything like that.
!!!{{{etc/packages}}}: the list of packages to stow
This is simply a list of package specifications, one per line. Skel doesn't really care how you name your packages, but it does have some mild support for versioned names and wildcards: If a package specification looks like a glob (wildcard filename) - in particular if it contains one of the characters {{{*}}}, {{{?}}}. or {{{[}}}, then it will be used to match against package names, and the newest matching package will be selected. The newest package is the one with the highest version number that matches: {{{3.0}}} is higher than {{{2.1}}} and {{{2.982.2}}}, while {{{3.0.1}}} is higher than {{{3.0}}}.
So for example if the [[package repository|missing]] contains {{{gcc-2.95}}}, {{{gcc-3.2}}}, {{{gcc-3.2.3}}} and {{{gcc-3.3.4}}}, then {{{gcc*}}} should match {{{gcc-3.3.4}}} while {{{gcc-2*}}} will match {{{gcc-2.95}}}.
It is probably better to specify packages as exactly as possible.
it is possible (by setting a parameter called {{{wildcardify-packages}}}) to have automatic package specification wildcardification, whereby {{{*}}} will be appended to packages which {{{cfskel}}} considers not to be fully specified. This is deprecated.
''Finding stow.'' One package must always be specified: stow itself. {{{cfskel}}} doesn't try and do anything clever such as automatically adding a suitable package specification for stow: you have to specify it in the file. {{{cfskel}}} will complain if there isn't a package that looks to it like stow, and it will check that there is a suitable stow executable in it. By default the stow package will be anything that matches the regular expression {{{/^stow.*/}}}. If you specify more than one package that matches this then the consequences are probably ill-defined.
''Example.'' This packages file is enough to deploy xenia - it also deploys a suitable python, and gcc to provide runtime library support for it.
{{{
# A package list which should be enough to deploy xenia
#
gcc-3.3.4
python-2.3.4
stow-1.3.3
xenia*
}}}
!!!{{{etc/package-configuration}}}: configuration for {{{cfskel}}}
This file contains parameters for {{{cfskel}}} (it's called {{{package-configuration}}} because it was originally intended to be more general). {{{cfskel}}} has a large number of named parameters, including directory locations and so on. These parameters can be given by command line switches, or if not then by the {{{package-configuration}}} file, and finally most have default values defined in {{{cfskel}}} itself.
There are rather a lot of parameters. Almost none of them are useful unless you want to do something strange like change the name of the default binary directory, and even then this may not be possible without changes to {{{cpskel}}} as well. So rather than document the parameters in detail here, I'll just give a couple of sample configuration files.
''Forcing {{{cfskel}}} to use absolute paths.'' This example sets the {{{use-absolute-paths}}} parameter, which will cause {{{cfskel}}} to generate symbolic links to absolute pathnames.
{{{
# use absolute paths
use-absolute-paths 1
}}}
The default is to use relative paths, which is generally better, except that it wires in the relative positions of the root and the package repository. {{{cfskel}}} has a special hack to avoid this:
{{{
# use relative paths
use-absolute-paths 0
# but absolute paths to the package repository
use-absolute-paths-to-package-repository 1
}}}
''Configuring the location of the package repository.'' This avoids the requirement to tell {{{cfskel}}} where the package repository is. This example also configures the symlinks as above.
{{{
# Sample package configuration
# use relative paths, but absolute for the repository
use-absolute-paths 0
use-absolute-paths-to-package-repository 1
# directory of the repository (which the program mostly calls
# package-dir...)
package-dir /home/tbradshaw/work/ps-prod/packages/
}}}
!!!{{{cfskel}}} usage
''Synopsis.'' {{{cfskel}}} has a lot of possible arguments. The following documents only the generally interesting ones.
{{{
cfskel [--verbose+]
[--debug]
[--use-absolute-paths] [--use-relative-paths]
[--package-dir package-dir]
[--parameter param=val]*
root
}}}
''Description''. {{{cfskel}}} configures an existing root. It:
* makes links to specified packages in the repository;
* stows the linked packages;
* wraps the application executables.
When {{{cfskel}}} has run the application should be usable.
* {{{--verbose}}}: if given this will increase the verbosity level. This argument can be repeated (currently only twice) to make the program more verbose. The second level is probably only interesting for debugging purposes - it is really fairly verbose.
* {{{--debug}}}: turn on debugging output. This is only interesting for development of {{{cfskel}}}.
* {{{--use-absolute-paths}}}: cause cfskel to create symlinks with absolute pathnames. The default is relative pathnames.
* {{{--use-relative-paths}}}: create symlinks with relative pathnames. This is the default.
* {{{--package-dir}}} //{{{package-dir}}}//: make //{{{package-dir}}}// be the package directory / package repository. Unless this is provided by the configuration file, this argument is mandatory.
* {{{--parameter}}} //{{{param}}}//{{{=}}}//{{{val}}}//: set //{{{param}}}// to //{{{val}}}//, overriding the configuration file and any default.
* //{{{root}}}//: the root to configure.
In almost all cases, {{{cfskel}}} will be run from the root it is configuring, although it can be run from the skeleton as well.
''Default directories.'' As well as the package directory, {{{cfskel}}} is interested in these directories, relative to the root.
* {{{etc/}}}: directory for package list and configuration files.
* {{{bin/}}}: directory containing application executables to wrap - all executable files with `reasonable' names (not starting with {{{.}}} for instance) in this directory will be wrapped.
!!!Example
Configuring the root created above. The two configuration files are:
{{{/home/tbradshaw/work/ps-prod/run/etc/packages}}}:
<<<
{{{
# A package list which should be enough to deploy xenia
#
gcc-3.3.4
python-2.3.4
stow-1.3.3
xenia*
}}}
<<<
{{{/home/tbradshaw/work/ps-prod/run/etc/packages-configuration}}}:
<<<
{{{
# Sample package configuration
# use relative paths, but absolute for the repository
use-absolute-paths 0
use-absolute-paths-to-package-repository 1
# directory of the repository (which the program mostly calls
# package-dir...)
package-dir /home/tbradshaw/work/ps-prod/packages/
}}}
<<<
Note: {{{package-configuration}}} defines {{{package-dir}}} so we don't strictly need to provide one. We do anyway.
{{{
$ ./run/sbin/cfskel -v --package-dir ./packages/ run/
User parameters after argument processing.
package-dir = "./packages/"
root-dir = "run/"
Defaulted parameters after reading run/etc/package-configuration.
_file = "run/etc/package-configuration"
bin-dir = "bin/"
bin-wrapper = "wrapper.pl"
config-file = "package-configuration"
etc-dir = "etc/"
package-dir = "./packages/"
package-file = "packages"
perl-wrapper = "perl-wrapper.sh"
root-dir = "run/"
sbin-dir = "sbin/"
stow-dir = "stow/"
use-absolute-paths = "0"
use-absolute-paths-to-package-repository = "1"
Packages.
gcc-3.3.4 -> gcc-3.3.4
python-2.3.4 -> python-2.3.4
stow-1.3.3 -> stow-1.3.3
xenia* -> xenia*
Linking.
gcc-3.3.4 is /home/tbradshaw/work/ps-prod/packages/gcc-3.3.4
python-2.3.4 is /home/tbradshaw/work/ps-prod/packages/python-2.3.4
stow-1.3.3 is /home/tbradshaw/work/ps-prod/packages/stow-1.3.3
xenia* is /home/tbradshaw/work/ps-prod/packages/xenia-0.0
Stowing in /home/tbradshaw/work/ps-prod/run/stow/.
stowing gcc-3.3.4
stowing python-2.3.4
stowing stow-1.3.3
stowing xenia-0.0
Wrapping in /home/tbradshaw/work/ps-prod/run/bin/.
wrap
g++
c++
i686-pc-linux-gnu-g++
i686-pc-linux-gnu-c++
gcov
gccbug
cpp
gcc
i686-pc-linux-gnu-gcc-3.3.4
i686-pc-linux-gnu-gcc
python2.3
pydoc
idle
python
stow
xenia
}}}
!!!Bugs
* There is no way of specifying the most useful combination of absolute and relative links from the command line (well: there is, but you have to use {{{--parameter}}} which is a pain).
* package dir or package repository? Originally dir, then the manual started calling it a repository but now I think dir is better.
* Configuration file reading is incoherent and inconsistent with the other bits of skel.
!!!Prerequisites
* {{{cfskel}}} needs a reasonable Unix platform.
* It needs Perl 5.6.0 or greater (it could probably be made to work with older Perls but I have not tested it, and it insists on 5.6.0.
The executable wrapper reads environment definition files for each wrapped executable. By default these files live in {{{etc}}} under the root (and so could be copied into place by {{{cfskel}}}). For a wrapped executable {{{bin/foo}}}, the wrapper will read {{{etc/environment}}} and then {{{etc/environment-foo}}} (so the executable-specific file gets to run last). It is OK for any of these files not to exist.
These files are in [[standard syntax|missing]], and contain a simple language for controlling the environment: variables can be set, cleared or added to, as well as checked for. The {{{ROOT}}} environment variable is set by the wrapper before these files are processed, and will be the absolute pathname of the root.
!!!Setting and clearing
{{{
set VAR value
}}}
Set the variable //{{{VAR}}}// to //{{{value}}}//. Environment substitutions are done on //{{{value}}}//. Example:
<<<
{{{
set PYTHONHOME ${ROOT}
}}}
<<<
{{{
maybe VAR value
}}}
Set the variable //{{{VAR}}}// to //{{{value}}}// if it is not already set. Environment substitutions are done on //{{{value}}}//. Example:
<<<
{{{
maybe PYTHONHOME ${ROOT}
}}}
<<<
{{{
clear VAR
}}}
Clear //{{{VAR}}}//. Example:
<<<
{{{
clear GRIBBLE # Avoid gribbling
}}}
<<<
!!!Appending and prepending
These commands append and prepend things to `path-type' variables, with a separator of {{{:}}}. They're useful for setting {{{PATH}}}, {{{LD_LIBRARY_PATH}}} and so on.
{{{
append VAR value
}}}
Append //{{{value}}}// to //{{{VAR}}}//: if //{{{VAR}}}// is unset it becomes //{{{value}}}//, if it is set it becomes {{{${}}}//{{{VAR}}}//{{{}:}}}//{{{value}}}//. Environment substitutions are done on //{{{value}}}//. Example:
<<<
{{{
append PATH ${ROOT}/other/bin
}}}
<<<
{{{
append-maybe VAR value
}}}
Maybe append value to //{{{VAR}}}//: if //{{{VAR}}}// is unset it becomes //{{{value}}}//, if its value contains the string //{{{value}}}// then do nothing, otherwise it becomes {{{${}}}//{{{VAR}}}//{{{}:}}}//{{{value}}}//. Environment substitutions are done on //{{{value}}}//. Example:
<<<
{{{
append-maybe PATH ${ROOT}/other/bin
}}}
<<<
{{{prepend}}} and {{{prepend-maybe}}} are like {{{append}}}, {{{append-maybe}}} but they prepend rather than appending.
There are also two commands, {{{append-pathsep}}} and {{{prepend-pathsep}}} that ensure that a `path-type' variable ends or starts with a path separator character. This is useful for variables such as {{{MANPATH}}}:
* {{{MANPATH=/my/dir/man}}} means `look for manual pages in the tree under {{{/my/dir/man}}} //only//';
* {{{MANPATH=/my/dir/man:}}} means `look for manual pages in {{{/my/dir/man}}}, and then search all the default locations'.
It's possible to ensure there's a suitable trailing path separator fairly reliably by using a combination of {{{maybe}}} and {{{append-maybe}}} (or {{{prepend-maybe}}}), but using {{{append-pathsep}}} ({{{prepend-pathsep}}}) is more succinct. A typical use might be as follows:
<<<
{{{
append-maybe MANPATH ${ROOT}/man
append-maybe MANPATH ${ROOT}/opt/jdk1.5.0/man
append-pathsep MANPATH
}}}
<<<
!!!Checking
These commands check the environment, and will cause the wrapper to abort if they aren't satisfied.
{{{
require VAR
}}}
require //{{{VAR}}}// to be set. Example:
<<<
{{{
require ROOT # should always be true!
}}}
<<<
{{{
forbid VAR
}}}
require //{{{VAR}}}// not to be set. Example:
<<<
{{{
forbid GRIB # can't hack this.
}}}
<<<
!!!A complete example environent file.
{{{
# Environment for all applications
# ROOT will always be set, but be paranoid.
require ROOT
# for our shared libs
prepend-maybe LD_LIBRARY_PATH ${ROOT}/lib
# for our applications
prepend-maybe PATH ${ROOT}/sbin
# for skel?
prepend-maybe PATH ${ROOT}/bin
# Manual pages
prepend-maybe MANPATH ${ROOT}/man
append-pathsep MANPATH
# for Python
set PYTHONHOME ${ROOT}
}}}
The wrapper doesn't read any configuration files, or take any arguments (it can't do that because it wants to pass its command line straight to the wrapped executable). It does listen to a couple of environment variables though.
* {{{WRAPPER_VERBOSE}}}: be verbose. If this is set the wrapper will talk about what it is doing (to standard error).
* {{{WRAPPER_VERBOSITY}}}: control verbosity in more detail.This can be one of:
** {{{0}}} - not verbose;
** {{{1}}} - same as {{{WRAPPER_VERBOSE}}};
** {{{2}}} - very verbose indeed.
* {{{WRAPPER_DEBUG}}}: turn on debugging. This will generate debugging output. It's probably not interesting unless you're debugging skel.
Here's an example of running a wrapped Python with {{{WRAPPER_VERBOSE}}}, using the environment definition file [[above|missing]].
{{{
$ WRAPPER_VERBOSE=yes ./run/bin/python
Found a root at /home/tbradshaw/work/ps-prod/run/
Considering environment /home/tbradshaw/work/ps-prod/run/etc/environment
Processing /home/tbradshaw/work/ps-prod/run/etc/environment
require ROOT
prepend-maybe LD_LIBRARY_PATH ${ROOT}/lib
prepend-maybe PATH ${ROOT}/sbin
prepend-maybe PATH ${ROOT}/bin
set PYTHONHOME ${ROOT}
Considering environment /home/tbradshaw/work/ps-prod/run/etc/environment-python
Python 2.3.4 (#1, Jun 23 2004, 15:39:43)
[GCC 3.3.4] on linux2
Type "help", "copyright", "credits" or "license" for more information.
>>>
}}}
There are some other environment variables which can be used to cause other environment definition files to be read and so on.
It's quite useful to be able to run a completely arbitrary command with the environment set in the same way as it would be for executables wrapped by {{{cfskel}}}. It's also often useful to be able to spawn an interactive shell with the environment set up. This is what {{{wrap}}} does. It's the only one of skel's commands that lives in {{{bin/}}} rather than {{{sbin/}}}.
''Example.'' wrapping the GNU autotools so it finds `our' version of gcc.
{{{
$ gcc -v
Reading specs from /lmn/tools/search/gcc/gcc-3.3.2/lib/gcc-lib/i686-pc-linux-gnu/3.3.2/specs
Configured with: ./configure --prefix=/lmn/tools/search/gcc/gcc-3.3.2 --enable-languages=c,c++
Thread model: posix
gcc version 3.3.2
$ ../../run/bin/wrap gcc -v
Reading specs from /home/tbradshaw/work/ps-prod/run/sbin/../lib/gcc-lib/i686-pc-linux-gnu/3.3.4/specs
Configured with: ../gcc-3.3.4/configure --prefix=/home/tbradshaw/work/ps-prod --enable-languages=c,c++
Thread model: posix
gcc version 3.3.4
$ ../../run/bin/wrap ./configure --prefix=/tmp/expat-1.95.8
[output elided]
}}}
If given no arguments, {{{wrap}}} just spawns a shell with the environment set up properly:
{{{
$ type python
python is /lmn/tools/search/python/python-2.3.3/bin/python
[tbradshaw@devbgb0210 ps-prod$ ./run/bin/wrap
$ type python
python is /home/tbradshaw/work/ps-prod/run/bin/python
$ exit
$
}}}
{{{wrap}}} is useful to doing things like builds and configuration of packages in a suitable environment, as well as running unwrapped commands.