diff -Nur --exclude=RCS --exclude=CVS --exclude=SCCS --exclude=BitKeeper --exclude=ChangeSet linux-2.5-import/arch/alpha/kernel/irq.c linux-2.5-usrdrivers/arch/alpha/kernel/irq.c --- linux-2.5-import/arch/alpha/kernel/irq.c Thu Oct 9 08:44:01 2003 +++ linux-2.5-usrdrivers/arch/alpha/kernel/irq.c Wed Nov 5 11:04:57 2003 @@ -218,6 +218,7 @@ } spin_unlock_irqrestore(&desc->lock,flags); + register_irq_proc(irq); return 0; } @@ -354,6 +355,128 @@ #endif /* CONFIG_SMP */ #define MAX_NAMELEN 10 +struct irq_proc { + struct semaphore sem; + int irq; + atomic_t count; + char devname[sizeof ((struct task_struct *)0)->comm]; +}; + +void +irq_proc_interrupt_handler(int irq, void *vidp, struct pt_regs *regs) +{ + struct irq_proc *idp = (struct irq_proc *)vidp; + + BUG_ON(idp->irq != irq); + disable_irq_nosync(irq); + atomic_inc(&idp->count); + up(&idp->sem); +} + +/* + * Read for the IRQ to happen then returns. + * If the interrupt is masked, enables it. + * Sleep until an interrupt happens, then + * returns 0 with the interrupt masked. + */ +ssize_t +irq_proc_read(struct file *fp, char *bufp, size_t len, loff_t *where) +{ + struct irq_proc *ip = (struct irq_proc *)fp->private_data; + irq_desc_t *idp = irq_desc + ip->irq; + int i; + int err; + + + if (len < sizeof(int)) + return -EINVAL; + if (idp->status & IRQ_DISABLED) + enable_irq(ip->irq); + for (;;) { + if ((i = atomic_read(&ip->count))) { + atomic_sub(i, &ip->count); + if ((err = copy_to_user(bufp, &i, sizeof(i)))) + return err; + *where += sizeof i; + return sizeof i; + } + if ((err = down_interruptible(&ip->sem))) + return err; + } + /* NOTREACHED */ +} + +ssize_t +irq_proc_write(struct file *fp, const char *bufp, size_t len, loff_t *where) +{ + struct irq_proc *ip = (struct irq_proc *)fp->private_data; + int i; + int err; + if (len < sizeof(int)) + return -EINVAL; + + if ((err = copy_from_user(&i, bufp, sizeof(i)))) + return err; + + if (i < 0) { + disable_irq_nosync(ip->irq); + } else if (i == 0) { + enable_irq(ip->irq); + } else { + atomic_sub(i, &ip->count); + } + *where += sizeof i; + return sizeof i; +} + + +int +irq_proc_open(struct inode *inp, struct file *fp) +{ + struct irq_proc *ip; + struct proc_dir_entry *ent = PDE(inp); + int error; + + ip = kmalloc(sizeof *ip, GFP_KERNEL); + if (ip == NULL) + return -ENOMEM; + memset(ip, sizeof(*ip), 0); + strncpy(ip->devname, current->comm, sizeof(ip->devname) -1); + sema_init(&ip->sem, 0); + atomic_set(&ip->count, 0); + ip->irq = (int)(unsigned long)ent->data; + fp->private_data = (void *)ip; + + if ((error = request_irq(ip->irq, + irq_proc_interrupt_handler, + SA_INTERRUPT, + ip->devname, + ip)) < 0) { + kfree(ip); + return error; + } + return 0; +} + +int irq_proc_release(struct inode *inop, struct file *fp) +{ + struct irq_proc *ip = (struct irq_proc *)fp->private_data; + + free_irq(ip->irq, ip); + fp->private_data = NULL; + kfree(ip); + return 0; +} + +struct file_operations irq_proc_file_operations = { + .read = irq_proc_read, + .write = irq_proc_write, + .open = irq_proc_open, + .release = irq_proc_release, + }; + +#define MAX_NAMELEN 10 + static void register_irq_proc (unsigned int irq) @@ -364,15 +487,29 @@ irq_dir[irq]) return; - memset(name, 0, MAX_NAMELEN); - sprintf(name, "%d", irq); - - /* create /proc/irq/1234 */ - irq_dir[irq] = proc_mkdir(name, root_irq_dir); - + if (!irq_dir[irq]) { + memset(name, 0, MAX_NAMELEN); + sprintf(name, "%d", irq); + + /* create /proc/irq/1234 */ + irq_dir[irq] = proc_mkdir(name, root_irq_dir); + + /* + * Create handles for user-mode interrupt handlers + * if the kernel hasn't already grabbed the IRQ + */ + entry = create_proc_entry("irq", 0600, irq_dir[irq]); + if (entry) { + entry->data = (void *)(unsigned long)irq; + entry->read_proc = NULL; + entry->write_proc = NULL; + entry->proc_fops = &irq_proc_file_operations; + + } + } #ifdef CONFIG_SMP - if (irq_desc[irq].handler->set_affinity) { struct proc_dir_entry *entry; + if (!smp_affinity_entry[irq] && irq_desc[irq].handler->set_affinity) { /* create /proc/irq/1234/smp_affinity */ entry = create_proc_entry("smp_affinity", 0600, irq_dir[irq]); @@ -418,8 +555,6 @@ */ if (ACTUAL_NR_IRQS < (PROC_NDYNAMIC / 4)) { for (i = 0; i < ACTUAL_NR_IRQS; i++) { - if (irq_desc[i].handler == &no_irq_type) - continue; register_irq_proc(i); } } diff -Nur --exclude=RCS --exclude=CVS --exclude=SCCS --exclude=BitKeeper --exclude=ChangeSet linux-2.5-import/arch/alpha/kernel/systbls.S linux-2.5-usrdrivers/arch/alpha/kernel/systbls.S --- linux-2.5-import/arch/alpha/kernel/systbls.S Tue Jul 15 08:48:02 2003 +++ linux-2.5-usrdrivers/arch/alpha/kernel/systbls.S Wed Nov 5 11:04:58 2003 @@ -444,7 +444,8 @@ .quad sys_clock_nanosleep .quad sys_semtimedop .quad sys_tgkill - + .quad usr_pci_map_sys + .quad usr_pci_open_sys .size sys_call_table, . - sys_call_table .type sys_call_table, @object diff -Nur --exclude=RCS --exclude=CVS --exclude=SCCS --exclude=BitKeeper --exclude=ChangeSet linux-2.5-import/arch/i386/kernel/entry.S linux-2.5-usrdrivers/arch/i386/kernel/entry.S --- linux-2.5-import/arch/i386/kernel/entry.S Thu Oct 2 09:52:06 2003 +++ linux-2.5-usrdrivers/arch/i386/kernel/entry.S Wed Nov 5 11:04:58 2003 @@ -880,5 +880,7 @@ .long sys_utimes .long sys_fadvise64_64 .long sys_ni_syscall /* sys_vserver */ + .long usr_pci_map_sys + .long usr_pci_open_sys /* 275 */ nr_syscalls=(.-sys_call_table)/4 diff -Nur --exclude=RCS --exclude=CVS --exclude=SCCS --exclude=BitKeeper --exclude=ChangeSet linux-2.5-import/arch/i386/kernel/irq.c linux-2.5-usrdrivers/arch/i386/kernel/irq.c --- linux-2.5-import/arch/i386/kernel/irq.c Mon Oct 27 08:59:13 2003 +++ linux-2.5-usrdrivers/arch/i386/kernel/irq.c Wed Nov 5 11:04:59 2003 @@ -45,6 +45,7 @@ #include #include #include +#include /* * Linux has a controller-independent x86 interrupt architecture. @@ -1023,30 +1024,163 @@ #define MAX_NAMELEN 10 +struct irq_proc { + struct semaphore sem; + int irq; + atomic_t count; + char devname[sizeof ((struct task_struct *)0)->comm]; +}; + +irqreturn_t irq_proc_interrupt_handler(int irq, void *vidp, struct pt_regs *regs) +{ + struct irq_proc *idp = (struct irq_proc *)vidp; + + BUG_ON(idp->irq != irq); + disable_irq_nosync(irq); + atomic_inc(&idp->count); + up(&idp->sem); + return IRQ_HANDLED; +} + +/* + * Read for the IRQ to happen then returns. + * If the interrupt is masked, enables it. + * Sleep until an interrupt happens, then + * returns 0 with the interrupt masked. + */ +ssize_t irq_proc_read(struct file *fp, char *bufp, size_t len, loff_t *where) +{ + struct irq_proc *ip = (struct irq_proc *)fp->private_data; + irq_desc_t *idp = irq_desc + ip->irq; + int i; + int err; + + + if (len < sizeof(int)) + return -EINVAL; + if (idp->status & IRQ_DISABLED) + enable_irq(ip->irq); + for (;;) { + if ((i = atomic_read(&ip->count))) { + atomic_sub(i, &ip->count); + if ((err = copy_to_user(bufp, &i, sizeof(i)))) + return err; + *where += sizeof i; + return sizeof i; + } + if ((err = down_interruptible(&ip->sem))) + return err; + } + /* NOTREACHED */ +} + +ssize_t irq_proc_write(struct file *fp, const char *bufp, size_t len, loff_t *where) +{ + struct irq_proc *ip = (struct irq_proc *)fp->private_data; + int i; + int err; + if (len < sizeof(int)) + return -EINVAL; + + if ((err = copy_from_user(&i, bufp, sizeof(i)))) + return err; + + if (i < 0) { + disable_irq_nosync(ip->irq); + } else if (i == 0) { + enable_irq(ip->irq); + } else { + atomic_sub(i, &ip->count); + } + *where += sizeof i; + return sizeof i; +} + + +int irq_proc_open(struct inode *inp, struct file *fp) +{ + struct irq_proc *ip; + struct proc_dir_entry *ent = PDE(inp); + int error; + + ip = kmalloc(sizeof *ip, GFP_KERNEL); + if (ip == NULL) + return -ENOMEM; + memset(ip, 0, sizeof(*ip)); + strncpy(ip->devname, current->comm, sizeof(ip->devname) -1); + sema_init(&ip->sem, 0); + atomic_set(&ip->count, 0); + ip->irq = (int)(unsigned long)ent->data; + + if ((error = request_irq(ip->irq, + irq_proc_interrupt_handler, + SA_INTERRUPT|SA_SHIRQ, + ip->devname, + ip)) < 0) { + kfree(ip); + return error; + } + fp->private_data = (void *)ip; + return 0; +} + +int irq_proc_release(struct inode *inop, struct file *fp) +{ + struct irq_proc *ip = (struct irq_proc *)fp->private_data; + + free_irq(ip->irq, ip); + fp->private_data = NULL; + kfree(ip); + return 0; +} + +struct file_operations irq_proc_file_operations = { + .read = irq_proc_read, + .write = irq_proc_write, + .open = irq_proc_open, + .release = irq_proc_release, +}; + +#define MAX_NAMELEN 10 + static void register_irq_proc (unsigned int irq) { char name [MAX_NAMELEN]; + struct proc_dir_entry *entry; - if (!root_irq_dir || (irq_desc[irq].handler == &no_irq_type) || - irq_dir[irq]) + if (!root_irq_dir || irq_desc[irq].handler == &no_irq_type) return; - memset(name, 0, MAX_NAMELEN); - sprintf(name, "%d", irq); + if (!irq_dir[irq]) { + memset(name, 0, MAX_NAMELEN); + sprintf(name, "%d", irq); - /* create /proc/irq/1234 */ - irq_dir[irq] = proc_mkdir(name, root_irq_dir); + /* create /proc/irq/1234 */ + irq_dir[irq] = proc_mkdir(name, root_irq_dir); + + /* + * Create handles for user-mode interrupt handlers + * if the kernel hasn't already grabbed the IRQ + */ + entry = create_proc_entry("irq", 0600, irq_dir[irq]); + if (entry) { + entry->data = (void *)(unsigned long)irq; + entry->read_proc = NULL; + entry->write_proc = NULL; + entry->proc_fops = &irq_proc_file_operations; + + } + } #ifdef CONFIG_SMP - { + if (!smp_affinity_entry[irq]) { struct proc_dir_entry *entry; - /* create /proc/irq/1234/smp_affinity */ entry = create_proc_entry("smp_affinity", 0600, irq_dir[irq]); if (entry) { entry->nlink = 1; - entry->data = (void *)(long)irq; + entry->data = (void *)(unsigned long)irq; entry->read_proc = irq_affinity_read_proc; entry->write_proc = irq_affinity_write_proc; } diff -Nur --exclude=RCS --exclude=CVS --exclude=SCCS --exclude=BitKeeper --exclude=ChangeSet linux-2.5-import/arch/ia64/kernel/entry.S linux-2.5-usrdrivers/arch/ia64/kernel/entry.S --- linux-2.5-import/arch/ia64/kernel/entry.S Wed Sep 10 08:43:13 2003 +++ linux-2.5-usrdrivers/arch/ia64/kernel/entry.S Wed Nov 5 11:05:00 2003 @@ -1473,8 +1473,8 @@ data8 sys_clock_nanosleep data8 sys_fstatfs64 data8 sys_statfs64 - data8 ia64_ni_syscall - data8 ia64_ni_syscall // 1260 + data8 usr_pci_map_sys + data8 usr_pci_open_sys // 1260 data8 ia64_ni_syscall data8 ia64_ni_syscall data8 ia64_ni_syscall diff -Nur --exclude=RCS --exclude=CVS --exclude=SCCS --exclude=BitKeeper --exclude=ChangeSet linux-2.5-import/arch/ia64/kernel/irq.c linux-2.5-usrdrivers/arch/ia64/kernel/irq.c --- linux-2.5-import/arch/ia64/kernel/irq.c Wed Nov 12 10:45:32 2003 +++ linux-2.5-usrdrivers/arch/ia64/kernel/irq.c Wed Nov 12 13:48:26 2003 @@ -1064,23 +1064,156 @@ return full_count; } +struct irq_proc { + struct semaphore sem; + int irq; + atomic_t count; + char devname[sizeof ((struct task_struct *) 0)->comm]; +}; + +irqreturn_t irq_proc_irq_handler(int irq, void *vidp, struct pt_regs *regs) +{ + struct irq_proc *idp = (struct irq_proc *)vidp; + + BUG_ON(idp->irq != irq); + disable_irq_nosync(irq); + atomic_inc(&idp->count); + up(&idp->sem); + return IRQ_HANDLED; +} + +/* + * Read for the IRQ to happen then returns. + * If the interrupt is masked, returns immediately with EBUSY; + * If the interrupt is unmasked, sleeps until the interrupt happens, then + * returns 0 with the interrupt masked. + * Note: no data is ever transferred to/from user space! + */ +ssize_t irq_proc_read(struct file *fp, char *bufp, size_t len, loff_t *where) +{ + struct irq_proc *ip = (struct irq_proc *)fp->private_data; + irq_desc_t *idp = irq_descp(ip->irq); + int i; + int err; + + if (len < sizeof(int)) + return -EINVAL; + if (idp->status & IRQ_DISABLED) + enable_irq(ip->irq); + for(;;) { + if ((i = atomic_read(&ip->count))) { + atomic_sub(i, &ip->count); + if ((err = copy_to_user(bufp, &len, sizeof(len)))) + return err; + *where += sizeof i; + return sizeof(len); + } + if ((err = down_interruptible(&ip->sem))) + return err; + } + /* NOTREACHED */ +} + +ssize_t irq_proc_write(struct file *fp, const char *bufp, size_t len, loff_t *where) +{ + struct irq_proc *ip = (struct irq_proc *)fp->private_data; + int i; + int err; + if (len < sizeof(int)) + return -EINVAL; + + if ((err = copy_from_user(&i, bufp, sizeof(i)))) + return err; + if (i < 0) { + disable_irq_nosync(ip->irq); + } else if (i == 0) { + enable_irq(i); + } else { + atomic_sub(i, &ip->count); + } + *where += sizeof i; + return sizeof i; +} + + +int irq_proc_open(struct inode *inop, struct file *fp) +{ + struct irq_proc *ip; + struct proc_dir_entry *ent = PDE(inop); + int error; + + ip = kmalloc(sizeof *ip, GFP_KERNEL); + if (ip == NULL) + return -ENOMEM; + + memset(ip, 0, sizeof(*ip)); + strcpy(ip->devname, current->comm); + sema_init(&ip->sem, 0); + atomic_set(&ip->count, 0); + ip->irq = (int)(unsigned long)ent->data; + + if ((error = request_irq(ip->irq, + irq_proc_irq_handler, + SA_INTERRUPT, + ip->devname, + ip)) < 0) { + kfree(ip); + return error; + } + fp->private_data = (void *)ip; + return 0; +} + +int irq_proc_release(struct inode *inop, struct file *fp) +{ + struct irq_proc *ip = (struct irq_proc *)fp->private_data; + + (void)inop; + free_irq(ip->irq, ip); + fp->private_data = NULL; + kfree(ip); + return 0; +} + +struct file_operations irq_proc_file_operations = { + .read = irq_proc_read, + .write = irq_proc_write, + .open = irq_proc_open, + .release = irq_proc_release, +}; + #define MAX_NAMELEN 10 static void register_irq_proc (unsigned int irq) { char name [MAX_NAMELEN]; + struct proc_dir_entry *entry; - if (!root_irq_dir || (irq_descp(irq)->handler == &no_irq_type) || irq_dir[irq]) + if (!root_irq_dir) return; - memset(name, 0, MAX_NAMELEN); - sprintf(name, "%d", irq); - - /* create /proc/irq/1234 */ - irq_dir[irq] = proc_mkdir(name, root_irq_dir); + if (!irq_dir[irq]) { + memset(name, 0, MAX_NAMELEN); + sprintf(name, "%d", irq); + + /* create /proc/irq/1234 */ + irq_dir[irq] = proc_mkdir(name, root_irq_dir); + + /* + * Create handles for user-mode interrupt handlers + * if the kernel hasn't already grabbed the IRQ + */ + entry = create_proc_entry("irq", 0600, irq_dir[irq]); + if (entry) { + entry->data = (void *)(unsigned long)irq; + entry->read_proc = NULL; + entry->write_proc = NULL; + entry->proc_fops = &irq_proc_file_operations; + } + } #ifdef CONFIG_SMP - { + if (!smp_affinity_entry[irq]) { struct proc_dir_entry *entry; /* create /proc/irq/1234/smp_affinity */ @@ -1123,8 +1256,6 @@ * Create entries for all existing IRQs. */ for (i = 0; i < NR_IRQS; i++) { - if (irq_descp(i)->handler == &no_irq_type) - continue; register_irq_proc(i); } } diff -Nur --exclude=RCS --exclude=CVS --exclude=SCCS --exclude=BitKeeper --exclude=ChangeSet linux-2.5-import/drivers/Makefile linux-2.5-usrdrivers/drivers/Makefile --- linux-2.5-import/drivers/Makefile Tue Sep 2 08:42:55 2003 +++ linux-2.5-usrdrivers/drivers/Makefile Wed Nov 5 11:05:01 2003 @@ -26,7 +26,7 @@ obj-$(CONFIG_SCSI) += scsi/ obj-$(CONFIG_FUSION) += message/ obj-$(CONFIG_IEEE1394) += ieee1394/ -obj-y += cdrom/ video/ +obj-y += cdrom/ video/ usr/ obj-$(CONFIG_MTD) += mtd/ obj-$(CONFIG_PCMCIA) += pcmcia/ obj-$(CONFIG_DIO) += dio/ diff -Nur --exclude=RCS --exclude=CVS --exclude=SCCS --exclude=BitKeeper --exclude=ChangeSet linux-2.5-import/drivers/pci/Kconfig linux-2.5-usrdrivers/drivers/pci/Kconfig --- linux-2.5-import/drivers/pci/Kconfig Fri Jun 13 09:35:30 2003 +++ linux-2.5-usrdrivers/drivers/pci/Kconfig Wed Nov 5 11:05:01 2003 @@ -35,3 +35,15 @@ When in doubt, say Y. + +config USRDEV + bool "Framework for user-mode PCI drivers" + depends on PCI + ---help--- + UNSW NICTA/Gelato user-mode device drivers. + +config USRBLKDEV + tristate "Framework for user-level block devices" + depends on USRDEV + ---help--- + UNSW NICTA/Gelato user-mode block devices. diff -Nur --exclude=RCS --exclude=CVS --exclude=SCCS --exclude=BitKeeper --exclude=ChangeSet linux-2.5-import/drivers/usr/Makefile linux-2.5-usrdrivers/drivers/usr/Makefile --- linux-2.5-import/drivers/usr/Makefile Thu Jan 1 10:00:00 1970 +++ linux-2.5-usrdrivers/drivers/usr/Makefile Wed Nov 12 13:44:08 2003 @@ -0,0 +1,2 @@ +obj-y += sys.o +obj-$(CONFIG_USRBLKDEV) += blkdev.o diff -Nur --exclude=RCS --exclude=CVS --exclude=SCCS --exclude=BitKeeper --exclude=ChangeSet linux-2.5-import/drivers/usr/blkdev.c linux-2.5-usrdrivers/drivers/usr/blkdev.c --- linux-2.5-import/drivers/usr/blkdev.c Thu Jan 1 10:00:00 1970 +++ linux-2.5-usrdrivers/drivers/usr/blkdev.c Wed Nov 5 11:05:01 2003 @@ -0,0 +1,807 @@ +/************************************************************************ + * blkdev.c -- user level block device loopback + * + * This file has two interfaces + * An interface to a user level block device driver, + * and a disk-like interface to the rest of the kernel. + * + * The user level block device interfaces to a file system. + * In the root of the file system are two directories: master and slave. + * + * Each user-mode disk device driver must create a file in master; + * a corresponding directory will be created automatically in slave. + * + * When the user-mode driver detects a disc, it writes an identify packet + * to the master file. This registers the disc. + * It's assumed that each master file will have at most four discs, two for + * each IDE channel. ###REVISIT FOR USER_MODE SCSI### + * + * When the user-mode-driver is idle, it does a read() operation from + * the master file. As elevator requests are removed from the queue + * and passed to the user-mode block devices, their addresses are + * converted to PCI addresses (by means of the IOMMU where necessary) + * and stuck into a scatterlist. + * + * If there are no waiting requests, the read will block. + * + * The user-mode driver is free to reorder requests if it likes. + * + * As each request is completed, the user-mode driver must write an + * IO-Complete packet, which removes the request from the outstanding + * operations queue and performs the in-kernel iocomplete routines. + * + * Each open/mount of a block device also increments a reference count on the + * master inode. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/*#define DEBUG 0*/ + + +#define BLKDEV_NPART 16 /* maximum number of partitions */ + +#define USR_BLKDEV_MAGIC (('B'<<24) | ('M'<<16) | ('G'<<8) | 'K') + +static struct vfsmount *usr_blkdev_mnt; +static struct dentry *usr_blkdev_root; +static struct dentry *slave; +static struct dentry *master; + +static int usr_blkdevopen(struct inode *, struct file *); +static ssize_t usr_blkdevread(struct file *, char *, size_t, loff_t *); +static ssize_t usr_blkdevwrite(struct file *, const char *, size_t, loff_t *); +static int usr_blkdevrelease(struct inode *, struct file *); + +struct usr_io_rqq { + struct usr_io_rqq *next; + struct request *reqp; + struct usr_io_req req; +}; + +struct usrblkdev { + struct semaphore readwait; /* for user-mode to sleep on */ + struct file *fp; /* reference to open file */ + + request_queue_t *queue; /* requests from elevator */ + spinlock_t qlock; /* lock for queue */ + + struct usr_io_rqq *outstanding; /* queue of outstanding requests */ + spinlock_t olock; + + + + struct pci_dev *pcidev; + int major; + struct gendisk *disks[4]; + struct dentry *master; + struct dentry *slave; + + spinlock_t alock; /* protect usecount and is_open */ + int usecount; /* number of users */ + int is_open; /* only one master operner allowd */ + struct file *pci_fp; /* corresponding file for usr_pci_fs */ +}; + +/* + * Master side opeartions. + * Slave side is invisible to us. + */ +static struct file_operations usr_blkdevfops = { + .open = usr_blkdevopen, + .read = usr_blkdevread, + .write = usr_blkdevwrite, + .release = usr_blkdevrelease, +}; + +/* + * Slave side operations + */ +static struct block_device_operations usr_slaveops = { + .owner = THIS_MODULE, +}; + +static struct dentry_operations usr_blkdev_dentry_operations = { +// .d_delete = usr_blkdev_delete_dentry, +}; + + +static int usr_blkdev_create(struct inode *, struct dentry *, int, struct nameidata *); +static int usr_blkdev_unlink(struct inode *,struct dentry *); +static int create_slave_blkdevs(struct usrblkdev *, int, int); +static int usr_blkdev_mknod(struct inode *dir, + struct dentry *dentry, int mode, dev_t dev); + +static struct inode_operations blkdev_dir_inode_operations = { + .create = usr_blkdev_create, + .unlink = usr_blkdev_unlink, + .lookup = simple_lookup, +}; + +static struct super_operations usr_blkdev_sops = { + .statfs = simple_statfs, +}; + +static void +usrblk_request(struct request_queue *q) +{ + struct usrblkdev *ub = (struct usrblkdev *)q->queuedata; + up(&ub->readwait); +} + +static int +create_slave_blkdevs(struct usrblkdev *up, int first_minor, int nparts) +{ + int i; + const int mode = S_IFBLK|0755; + int err; + + struct dentry *dp; + + for (i = first_minor; i < first_minor + nparts; i++){ + char buf[8]; + dp = lookup_one_len(buf, up->slave, sprintf(buf, "%d", i)); + if (IS_ERR(dp)) + return PTR_ERR(dp); + err = usr_blkdev_mknod(up->slave->d_inode, dp, mode, MKDEV(up->major, i)); + if (err) { + dput(dp); + return err; + } + } + return 0; +} + +/* + * Called when the + * master side is opened. + */ +static int +usr_blkdevopen(struct inode *ino, struct file *fp) +{ + struct usrblkdev *ub; + int ret = 0; + + if ((ub = ino->u.generic_ip)) { + spin_lock(&ub->alock); + if (ub->is_open) + ret = -EBUSY; + else + ub->is_open = 1; + spin_unlock(&ub->alock); + if (ret) + return ret; + fp->private_data = ub; + return ret; + } + return -EINVAL; +} + +/* + * Master side read operation. + */ +static ssize_t +usr_blkdevread(struct file *fp, char *buf, size_t len, loff_t *offset) +{ + struct usrblkdev *up = fp->private_data; + unsigned long flags; + struct request *req; + struct usr_io_rqq *rqq; + struct usr_io_req *iop; + struct gendisk *disk; + int res; + int nbytes; + int totalbytes = 0; + + spin_lock_irqsave(&up->qlock, flags); + again: + while (NULL == (req = elv_next_request(up->queue))) { + spin_unlock_irqrestore(&up->qlock, flags); + if (down_interruptible(&up->readwait)) + return -EINTR; + spin_lock_irqsave(&up->qlock, flags); + } + res = 0; + blkdev_dequeue_request(req); + if (!(req->flags & REQ_CMD)) + goto out; + + disk = req->rq_disk; + if (req->sector + req->nr_sectors > disk->capacity) + goto out; + spin_unlock_irqrestore(&up->qlock, flags); + +#ifdef DEBUG + printk("req = %p, nr_sectors=%lu current_nr_sectors=%u hard_sector %llu hard_nr_sectors %lu\n", req, + req->nr_sectors, + req->current_nr_sectors, + (unsigned long long)req->hard_sector, + req->hard_nr_sectors); +#endif + nbytes = sizeof *iop + (req->nr_phys_segments * sizeof(struct scatterlist)); + if (nbytes > len) + return totalbytes == 0 ? -EINVAL : totalbytes; + + /* + * Build a request and send it to the user... + */ + + + rqq = kmalloc(nbytes - sizeof (struct usr_io_req) + sizeof (struct usr_io_rqq), GFP_KERNEL); + rqq->reqp = req; + iop = &rqq->req; + iop->raw_nentries = blk_rq_map_sg(up->queue, req, iop->sglist); +#ifdef DEBUG + printk("mapping SG: iop %p rqq %p device %p, sglist %p, nentries %d\n", + iop, rqq, up->pcidev, iop->sglist, iop->raw_nentries); +#endif + iop->nentries = pci_map_sg(up->pcidev, iop->sglist, + iop->raw_nentries, + rq_data_dir(req) == READ ? + PCI_DMA_FROMDEVICE : PCI_DMA_TODEVICE); +#ifdef DEBUG + printk("mapping: 1st entry: bus %p, virt %p\n", + (char *)iop->sglist[0].dma_address, + __va(page_to_phys(iop->sglist[0].page))); +#endif + iop->cmd = rq_data_dir(req)== READ ? + blk_read : blk_write; + iop->nsectors = req->nr_sectors; + iop->startsector = req->sector; + iop->tag = (__u64)(unsigned long)iop; + iop->unit = disk->first_minor / BLKDEV_NPART; + + spin_lock_irqsave(&up->olock, flags); + rqq->next = up->outstanding; + up->outstanding = rqq; + spin_unlock_irqrestore(&up->olock, flags); + + nbytes = (char *)&iop->sglist[iop->nentries] - (char *)iop; + + len -= nbytes; + totalbytes += nbytes; + if (copy_to_user(buf, iop, nbytes)) + return -EFAULT; + /* + * TODO: fetch more than one request at a time, if possible + */ + return totalbytes; + + out: +#ifdef DEBUG + printk("end_that_request_first(ERROR)\n"); +#endif + if (!end_that_request_first(req, 0, req->nr_sectors)) { + end_that_request_last(req); + } + goto again; + +} + +static int blk_complete_register(void *arg) +{ + struct gendisk *dp = (struct gendisk *)arg; + +#ifdef DEBUG + printk("blk_complete_register: about to add disk %s\n", dp->disk_name); +#endif + add_disk(dp); + create_slave_blkdevs(dp->private_data, dp->first_minor, BLKDEV_NPART); + + return 0; +} + +/* + * User mode has scanned the I/O-side bus and found a disc. + * Create the minor devices + * TODO: devolve to separate thread to allow user mode to respond to read + * requests during add_disk (to get at partitioning info). + */ +static int +usr_blkdev_add_disc(struct usrblkdev *up, struct blk_identify *idp) +{ + struct gendisk *disk; + + printk(KERN_INFO "usrblkdev: adding channel %d: %s; capacity %llu sectors of size %u; total capacity %lu MB\n", + idp->unit/2, idp->unit & 1 ? "slave" : "master", + (unsigned long long)idp->capacity, + idp->sectsize, + ((unsigned long)idp->capacity - (unsigned long)idp->capacity / 625 + 974) / 1950); + + if (idp->unit > 3) + return -EINVAL; + + if (up->disks[idp->unit]) { + printk(KERN_WARNING "Device already initialised\n"); + return -EBUSY; + } + disk = up->disks[idp->unit] = alloc_disk(BLKDEV_NPART); + if (!disk) + return -ENOMEM; + disk->major = up->major; + disk->first_minor = BLKDEV_NPART * idp->unit; + strncpy(disk->disk_name, up->master->d_name.name, sizeof disk->disk_name); + disk->fops = &usr_slaveops; + set_capacity(disk, idp->capacity); + disk->private_data = up; + disk->queue = up->queue; + + /* FIXME make a workqueue */ + kernel_thread(blk_complete_register, disk, CLONE_KERNEL); + + return 0; +} + +static int +usr_blkdev_iocomplete(struct usrblkdev *up, struct blk_iocomplete *icp) +{ + unsigned long flags; + struct usr_io_rqq **rqq; + struct usr_io_rqq *req; + struct request *rq; + + spin_lock_irqsave(&up->olock, flags); + + rqq = &up->outstanding; + while (*rqq && icp->tag != (__u64)(unsigned long)&(*rqq)->req) + rqq = &(*rqq)->next; + if (*rqq == NULL) { + spin_unlock_irqrestore(&up->olock, flags); + + printk("Tag %llx not found!\n", + (unsigned long long)icp->tag); + return -EINVAL; + } + req = *rqq; + *rqq = (*rqq)->next; + rq = req->reqp; + spin_unlock_irqrestore(&up->olock, flags); + +#ifdef DEBUG + printk("IOCOMPLETE: Found tag %llx, request is %p\n", icp->tag, req); + printk("IOCOMPLETE: unmapping SG: device %p, sglist %p, nentries %d\n", + up->pcidev, req->req.sglist, req->req.raw_nentries); +#endif + pci_unmap_sg(up->pcidev, req->req.sglist, req->req.raw_nentries, rq_data_dir(rq)); + + spin_lock_irqsave(&up->qlock, flags); + +#ifdef DEBUG + printk("IOCOMPLETE: errcode = %d; hard_cur_sectors = %d\n", + icp->errcode, (int)rq->hard_cur_sectors); + printk("end_that_request_first(%s)\n", icp->errcode ? "ERROR": "OK"); +#endif + if (!end_that_request_first(rq, !icp->errcode, rq->nr_sectors)) { +#ifdef DEBUG + printk("end_that_request_last\n"); +#endif + end_that_request_last(rq); + } + spin_unlock_irqrestore(&up->qlock, flags); + kfree(req); + return 0; +} + +static int +usr_blkdev_set_pci(struct usrblkdev *up, struct blk_pci *pp) +{ + struct pci_dev *pci_devp; + struct file *fp; + int err; + + err = usr_pci_get_dev(pp->fd, &pci_devp, &fp); + if (err) + return err; + if (up->fp != NULL) + usr_pci_put_dev(up->fp); + up->pcidev = pci_devp; + up->fp = fp; + return 0; +} + +static ssize_t +usr_blkdevwrite(struct file *fp, const char *p, size_t len, loff_t *off) +{ + struct usrblkdev *up = fp->private_data; + struct usr_io_complete uioc; + + if (copy_from_user(&uioc.what, p, sizeof uioc.what)) + return -EFAULT; + len -= sizeof uioc.what; + p += sizeof uioc.what; + switch (uioc.what) + { + case blk_identify: + if (len < sizeof uioc.u.ident) + return -EINVAL; + if (copy_from_user(&uioc.u.ident, p, sizeof uioc.u.ident)) + return -EFAULT; + return usr_blkdev_add_disc(up, &uioc.u.ident); + + case blk_io_complete: + if (len < sizeof uioc.u.err) + return -EINVAL; + if (copy_from_user(&uioc.u.err, p, sizeof uioc.u.err)) + return -EFAULT; + return usr_blkdev_iocomplete(up, &uioc.u.err); + + case blk_io_pci: + if (len < sizeof(uioc.u.pcifd)) + return -EINVAL; + if (copy_from_user(&uioc.u.pcifd, p, sizeof uioc.u.pcifd)) + return -EFAULT; + return usr_blkdev_set_pci(up, &uioc.u.pcifd); + + default: + printk("USR_BLKDEV: got cmd %d\n", (int)uioc.what); + return -EINVAL; + } +} + + +/* + * Called on last close + */ +static int +usr_blkdevrelease(struct inode *ino, struct file *fp) +{ + struct usrblkdev *ub; + struct usr_io_rqq *rq; + int i; + struct request *req; + unsigned long flags; + + + ub = ino->u.generic_ip; + BUG_ON(!ub); + BUG_ON(!ub->is_open); + + /* Don't allow opens while we're closing */ + spin_lock_irqsave(&ub->alock, flags); + + /* + * End all outstanding requests (-EINTR if possible) + */ + spin_lock(&ub->olock); + for (rq = ub->outstanding; rq; rq = ub->outstanding) { + ub->outstanding = rq->next; + req = rq->reqp; + spin_lock(&ub->qlock); +#ifdef DEBUG + printk("end_that_request_first(ERROR)\n"); +#endif + if (!end_that_request_first(req, 0, req->hard_cur_sectors)) { +#ifdef DEBUG + printk("end_that_request_last\n"); +#endif + end_that_request_last(req); + } + spin_unlock(&ub->qlock); + kfree(rq); + } + spin_unlock(&ub->olock); + ub->is_open = 0; + spin_unlock_irqrestore(&ub->alock, flags); + if (ub->fp) { + usr_pci_put_dev(ub->fp); + ub->fp = NULL; + ub->pcidev = NULL; + } + /* + * release all disks + */ + for (i = 0; i < 4; i++) + if (ub->disks[i]) { + del_gendisk(ub->disks[i]); + put_disk(ub->disks[i]); + ub->disks[i] = NULL; + } + return 0; +} + + + +/* + * Get and fill in an inode + */ +static struct inode * +usr_blkdev_get_inode(struct super_block *sb, + int mode, dev_t dev) +{ + struct inode * inode = new_inode(sb); + + if (inode) { + inode->i_mode = mode; + inode->i_uid = current->fsuid; + inode->i_gid = current->fsgid; + inode->i_blksize = PAGE_CACHE_SIZE; + inode->i_blocks = 0; + inode->i_rdev = 0; + inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME; + switch (mode & S_IFMT) { + default: + init_special_inode(inode, mode, dev); + break; + case S_IFREG: + inode->i_fop = &usr_blkdevfops; + break; + case S_IFDIR: + inode->i_op = &simple_dir_inode_operations; + inode->i_fop = &simple_dir_operations; + + /* directory inodes start off with i_nlink == 2 (for "." entry) */ + inode->i_nlink++; + break; + + + case S_IFLNK: + inode->i_op = &page_symlink_inode_operations; + break; + } + } + return inode; +} + + + +/* + * File creation. Allocate an inode, and we're done.. + */ +/* SMP-safe */ +static int usr_blkdev_mknod(struct inode *dir, + struct dentry *dentry, int mode, dev_t dev) +{ + struct inode *inode; + int error = -ENOSPC; + + BUG_ON(!dentry); + if (dentry->d_inode) + return -EEXIST; + + inode = usr_blkdev_get_inode(dir->i_sb, mode, dev); + + if (inode) { + d_instantiate(dentry, inode); + dget(dentry); /* Extra count - pin the dentry in core */ + error = 0; + } + return error; +} + + +/* + * `Ordinary' file creation in the master directory. + * We check to make sure we're not in the slave directory or subdirs (that's + * not allowed) and create the corresponding slave directory. + * ip points to directory Inode; + * dentry is new file. + */ +static int +usr_blkdev_create(struct inode *ip, struct dentry *dp, int mode, struct nameidata *nid) +{ + struct usrblkdev *ub; + struct dentry *slvdir; + int error; + + if (ip != master->d_inode) + return -EPERM; + + ub = kmalloc(sizeof(struct usrblkdev), GFP_KERNEL); + memset(ub, 0, sizeof (struct usrblkdev)); + + sema_init(&ub->readwait, 0); + + + spin_lock_init(&ub->qlock); + spin_lock_init(&ub->olock); + spin_lock_init(&ub->alock); + ub->is_open = 0; + + ub->major = register_blkdev(0, dp->d_name.name); + + if (ub->major == 0) { + error = -EAGAIN; + goto err0; + } + + ub->queue = blk_init_queue(usrblk_request, &ub->qlock); + if (ub->queue == NULL) { + error = -ENOMEM; + goto err1; + } + ub->queue->queuedata = ub; + + + error = usr_blkdev_mknod(ip, dp, mode, 0); + if (error) { + goto err2; + } + + dp->d_inode->u.generic_ip = (void *)ub; + ub->master = dp; + + slvdir = lookup_one_len(dp->d_name.name, slave, dp->d_name.len); + + if (IS_ERR(slvdir)) { + error = PTR_ERR(slvdir); + goto err3; + } + + + mode &= ~S_IFMT; + error = usr_blkdev_mknod(slave->d_inode, slvdir, mode|S_IFDIR, 0); + if (error) + goto err4; + + slvdir->d_fsdata = dp; + dp->d_fsdata = slvdir; + ub->slave = slvdir; + + return 0; + + + err4: + dput(slvdir); + err3: + dput(dp); + err2: + blk_cleanup_queue(ub->queue); + err1: + unregister_blkdev(ub->major, dp->d_name.name); + err0: + kfree(ub); + return error; +} + + +/* + * Allow unlink of master iff all slaves have already been unlinked. + * TODO: make unlink of master automatic when last slave is gone; + * attempt to demove slave(s) when master goes. + */ +static int +usr_blkdev_unlink(struct inode *dir,struct dentry *den) +{ + struct inode *ip = den->d_inode; + if (ip != master->d_inode) { + if (S_ISDIR(ip->i_mode)) + ((struct dentry *)den->d_fsdata)->d_fsdata = NULL; + ip->i_nlink--; + dput(den); + dput(den); + return 0; + } + BUG_ON(!S_ISREG(ip->i_mode)); + if (den->d_fsdata) + return -EINVAL; + + ip->i_nlink--; + dput(den); + dput(den); + return 0; +} + +static int +usr_blkdev_fill_super(struct super_block *s, void *data, int silent) +{ + struct inode * inode; + + s->s_blocksize = 1024; + s->s_blocksize_bits = 10; + s->s_magic = USR_BLKDEV_MAGIC; + s->s_op = &usr_blkdev_sops; + + inode = new_inode(s); + if (!inode) + goto fail; + inode->i_ino = 1; + inode->i_mtime = inode->i_atime = inode->i_ctime = CURRENT_TIME; + inode->i_blocks = 0; + inode->i_blksize = 1024; + inode->i_uid = inode->i_gid = 0; + inode->i_mode = S_IFDIR | S_IRUGO | S_IXUGO | S_IWUSR; + inode->i_op = &simple_dir_inode_operations; + inode->i_fop = &simple_dir_operations; + inode->i_nlink = 2; + + usr_blkdev_root = s->s_root = d_alloc_root(inode); + if (s->s_root) + return 0; + + printk("usr_blkdev: get root dentry failed\n"); + iput(inode); +fail: + return -ENOMEM; +} + + + +static struct super_block * +usr_blkdev_getsb( + struct file_system_type *fstype, + int flags, + const char *dev_name, + void *data) +{ + return get_sb_single(fstype, flags, data, usr_blkdev_fill_super); +} + + + +static struct file_system_type usr_blkdev_fs_type = { + .owner = THIS_MODULE, + .name = "usr_blkdevfs", + .get_sb = usr_blkdev_getsb, + .kill_sb = kill_anon_super, +}; + + + + +static int __init +init_usr_blkdev(void) +{ + int err = register_filesystem(&usr_blkdev_fs_type); + struct dentry *root; + + if (err) { + printk("Couldn't register usr_blkdev_fs: %d\n", err); + return err; + } + usr_blkdev_mnt = kern_mount(&usr_blkdev_fs_type); + err = PTR_ERR(usr_blkdev_mnt); + if (IS_ERR(usr_blkdev_mnt)) { + printk("Couldn't kern_mount usr_blkdev_fs; %d\n", err); + return err; + } + root = usr_blkdev_root; + down(&root->d_inode->i_sem); + master = lookup_one_len("master", root, 6); + usr_blkdev_mknod(root->d_inode, master, S_IFDIR|0755, 0); + master->d_inode->i_op = &blkdev_dir_inode_operations; + slave = lookup_one_len("slave", root, 5); + usr_blkdev_mknod(root->d_inode, slave, S_IFDIR|0555, 0); + up(&root->d_inode->i_sem); + + + /* + * Allow unloading... + */ + module_put(THIS_MODULE); + return 0; +} + +static void __exit +exit_usr_blkdev(void) +{ +#ifdef DEBUG + printk("exit_usr_blkdev called\n"); +#endif + unregister_filesystem(&usr_blkdev_fs_type); + + /* + * Release all dentries, starting from slaves + */ + mntput(usr_blkdev_mnt); +} + +module_init(init_usr_blkdev); +module_exit(exit_usr_blkdev); +MODULE_LICENSE("GPL"); + diff -Nur --exclude=RCS --exclude=CVS --exclude=SCCS --exclude=BitKeeper --exclude=ChangeSet linux-2.5-import/drivers/usr/sys.c linux-2.5-usrdrivers/drivers/usr/sys.c --- linux-2.5-import/drivers/usr/sys.c Thu Jan 1 10:00:00 1970 +++ linux-2.5-usrdrivers/drivers/usr/sys.c Wed Nov 5 11:05:01 2003 @@ -0,0 +1,721 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include /* virt_to_phys() on alpha */ + +#ifdef CONFIG_USRDEV + +/* + * For mappings < KMALLOC_BREAKPOINT pages, keep a cache; + * otherwise use kmalloc/kfree + */ +#define KMALLOC_BREAKPOINT 10 + + +#define USR_PCI_MAGIC (0x12345678) + +static int usr_pci_release(struct inode *ip, struct file *fp); + +static int +usr_pci_delete_dentry(struct dentry *dp) +{ + (void)dp; + return 1; +} + + +static struct vfsmount *usr_pci_mnt; +static struct file_operations usr_pci_fops = { + .release = usr_pci_release, +}; + +static struct dentry_operations usr_pci_dentry_operations = { + .d_delete = usr_pci_delete_dentry, +}; + +static struct super_block *usr_pci_getsb( + struct file_system_type *fstype, + int flags, + const char *dev_name, + void *data) +{ + return get_sb_pseudo(fstype, "usr_pci:", NULL, USR_PCI_MAGIC); +} + +static struct file_system_type usr_pci_fs_type = { + .name = "usr_pci_fs", + .get_sb = usr_pci_getsb, + .kill_sb = kill_anon_super, +}; + + +/* + * Describe a block of PCI-consistent memory mapped by the usr_map_pci() + * system call + */ +struct consistent_mem { + struct pci_dev *devp; + unsigned long len; + unsigned long kaddr; + unsigned long dmaaddr; +}; + +/* + * A chain of free struct internal_mapping_info + * are kept as a cache, and a chain of in-use ones are chained from the + * open file descriptor. + */ +struct internal_mapping_info { + struct internal_mapping_info *next; + struct mapping_info m; + unsigned nent; /* number of entries in sg[] */ + unsigned npages; /* number of entries in pages[] */ + struct scatterlist *sg; + struct page *pages[1]; +}; + +/* + * The private open-file data. + * --- the list of struct internal_mapping_info, + * a lock to protect list, + * and a pointer to the PCI device we're managing. + */ +struct maplist { + struct pci_dev *dev; + spinlock_t lk; /* protect the mappings list */ + struct internal_mapping_info *mappings; +}; + + +static void usr_pci_unmapconsistent(struct vm_area_struct *vma); + +struct vm_operations_struct usr_pci_vmop = { + .close = usr_pci_unmapconsistent, +}; + +/* + * Free list of mappings, and a spinlock to protect it. + */ +static struct internal_mapping_info *imap_free; +static spinlock_t imap_free_lock = SPIN_LOCK_UNLOCKED; + +/* + * get/free imap: + * allocate and initialise an internal_mapping_info structure + * or free one. + * + * If the mapping is for fewer than KMALLOC_BREAKPOINT pages, + * keep a cache of mapping infos. + * + */ +struct internal_mapping_info * +get_imap(int npages) +{ + unsigned long flags; + struct internal_mapping_info *ip = NULL; + if (imap_free && npages <= KMALLOC_BREAKPOINT) { + spin_lock_irqsave(&imap_free_lock, flags); + if (imap_free) { + ip = imap_free; + imap_free = ip->next; + } + spin_unlock_irqrestore(&imap_free_lock, flags); + } + + if (ip == NULL) { + if (npages < KMALLOC_BREAKPOINT) + npages = KMALLOC_BREAKPOINT; + ip = kmalloc(sizeof (*ip) + npages * sizeof (struct page *) + npages * sizeof(struct scatterlist), GFP_KERNEL); + } + + if (ip) { + ip->next = NULL; + ip->sg = (struct scatterlist *)&ip->pages[npages]; + ip->npages = 0; + } + + return ip; +} + +static void inline +free_imap(struct internal_mapping_info *ip) +{ + int i; + unsigned long flags; + + for (i = 0; i < ip->npages; i++) + if (!PageReserved(ip->pages[i])) + page_cache_release(ip->pages[i]); + if (ip->npages > KMALLOC_BREAKPOINT) { + kfree(ip); + return; + } + + spin_lock_irqsave(&imap_free_lock, flags); + ip->next = imap_free; + imap_free = ip; + spin_unlock_irqrestore(&imap_free_lock, flags); +} + +/* + * Same as for get/free_imap, but for low-level scatter gather lists. + * to be passed to the user-mode driver. + */ +static void *hw_free; +static spinlock_t hw_free_lock = SPIN_LOCK_UNLOCKED; + +static struct usr_pci_sglist * +get_hw(int n) +{ + struct usr_pci_sglist *p; + unsigned long flags; + + if (hw_free && n <= KMALLOC_BREAKPOINT) { + spin_lock_irqsave(&hw_free_lock, flags); + if ((p = hw_free)) + hw_free = *(void **)p; + spin_unlock_irqrestore(&hw_free_lock, flags); + if (p) + return p; + } + p = kmalloc((sizeof *p) * (n >= KMALLOC_BREAKPOINT ? n : KMALLOC_BREAKPOINT), GFP_KERNEL); + return p; +} + +static void +free_hw(int n, struct usr_pci_sglist *p) +{ + unsigned long flags; + if (n > KMALLOC_BREAKPOINT) { + kfree(p); + return; + } + spin_lock_irqsave(&hw_free_lock, flags); + *(void **)p = hw_free; + hw_free = (void *)p; + spin_unlock_irqrestore(&hw_free_lock, flags); +} + +/* + * Map a kernel page into user memory. + * Mark the space as non-cached and reserved. + */ +static void * +do_map(void *kaddr, + unsigned long origlen, + unsigned long dmaaddr, + struct pci_dev *devp) +{ + struct vm_area_struct *vma; + struct mm_struct *mm = current->mm; + unsigned long addr; + unsigned long len = origlen; + struct consistent_mem *mp; + + + vma = kmem_cache_alloc(vm_area_cachep, SLAB_KERNEL); + if (!vma) + return (void *)-ENOMEM; + mp = kmalloc(sizeof(*mp), GFP_KERNEL); + if (!mp){ + kmem_cache_free(vm_area_cachep, vma); + return (void *)-ENOMEM; + } + + mp->devp = devp; + mp->kaddr = (unsigned long)kaddr; + mp->dmaaddr = dmaaddr; + mp->len = origlen; + + vma->vm_mm = mm; + vma->vm_flags = VM_WRITE|VM_MAYWRITE|VM_READ|VM_MAYREAD|VM_RESERVED|VM_DONTCOPY; + vma->vm_flags |= VM_SHM|VM_LOCKED; + vma->vm_page_prot = PAGE_SHARED; + // vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); + vma->vm_ops = &usr_pci_vmop; + vma->vm_pgoff = 0; + vma->vm_file = NULL; + vma->vm_private_data = mp; + INIT_LIST_HEAD(&vma->shared); + + down_write(&mm->mmap_sem); + addr = get_unmapped_area(NULL, 0, len, 0, MAP_LOCKED|MAP_SHARED); + if (addr == (unsigned long)-ENOMEM) { + up_write(&mm->mmap_sem); + kmem_cache_free(vm_area_cachep, vma); + kfree(mp); + return (void *)-ENOMEM; + } + + vma->vm_start = addr; + vma->vm_end = addr + len; + + insert_vm_struct(mm, vma); + + while (len >= PAGE_SIZE) { + long page = virt_to_phys(kaddr); + if (remap_page_range(vma, addr, page, PAGE_SIZE, PAGE_SHARED)) { + up_write(&mm->mmap_sem); + do_munmap(mm, vma->vm_start, origlen); + return (void *)-ENOMEM; + } + addr += PAGE_SIZE; + kaddr += PAGE_SIZE; + len -= PAGE_SIZE; + } + + up_write(&mm->mmap_sem); + return (void *)vma->vm_start; +} + + +/* + * Undo the mapping into user space. + * This routine will be called from the vma infrastructure's teardown on + * munmap or process exit. + */ +static void +usr_pci_unmapconsistent(struct vm_area_struct *vma) { + struct consistent_mem *mp = (struct consistent_mem *)vma->vm_private_data; + vma->vm_private_data = NULL; + + ClearPageReserved(virt_to_page(mp->kaddr)); + pci_free_consistent(mp->devp, mp->len, (void *)mp->kaddr, mp->dmaaddr); + kfree(mp); +} + +/* + * The usr_pci_map system call. + */ +int asmlinkage +usr_pci_map_sys(int fd, int action, struct mapping_info *mp) +{ + struct mapping_info m; + struct maplist *p; + struct internal_mapping_info *imp; + unsigned long flags; + int i; + int npages; + struct file *filp = fget(fd); + struct pci_dev *devp; + + if (!filp) + return -EINVAL; + + if (filp->f_op != &usr_pci_fops) { + fput(filp); + return -EINVAL; + } + + if (copy_from_user(&m, mp, sizeof m)) { + fput(filp); + return -EFAULT; + } + p = (struct maplist *)filp->private_data; + fput(filp); + + if (!p) { + printk(KERN_ERR "usr_pci_fs file has no PCI device???"); + return -EINVAL; + } + devp = p->dev; + + + switch (action) + { + case USR_ALLOC_CONSISTENT: + { + dma_addr_t dma_handle; + void *vaddr; + + if (m.size != PAGE_SIZE) + return -EINVAL; + vaddr = pci_alloc_consistent(devp, m.size, &dma_handle); + if (vaddr == NULL) + return -ENOMEM; + + SetPageReserved(virt_to_page(vaddr)); + + m.dmaaddr = dma_handle; + m.virtaddr = do_map(vaddr, m.size, dma_handle, devp); + + if (((unsigned long)m.virtaddr) & ~PAGE_MASK) { + ClearPageReserved(virt_to_page(vaddr)); + pci_free_consistent(devp, m.size, vaddr, dma_handle); + return (unsigned long)m.virtaddr; + } + copy_to_user(mp, &m, sizeof m); + + return 0; + } + + case USR_MAP: + { + struct usr_pci_sglist *hw; + struct scatterlist *sgp; + int write; + int maxpages = m.size / PAGE_SIZE + 2; + + switch (m.direction) { + case DMA_TO_DEVICE: + write = 0; + break; + case DMA_FROM_DEVICE: + case DMA_BIDIRECTIONAL: + write = 1; + break; + default: + return -EINVAL; + } + if ((m.nents < maxpages) || m.size == 0) + return -EINVAL; + + imp = get_imap(maxpages); + if (imp == NULL) + return -ENOMEM; + + if ((npages = get_user_pages(current, + current->mm, + (unsigned long)m.virtaddr, + maxpages, + write, + 0, + imp->pages, + NULL)) < 0) { + free_imap(imp); + return npages; + } + + imp->npages = npages; + imp->m = m; + + if (npages < m.size/PAGE_SIZE){ + /* + * Not all the requested area is mapped. + * Return -EFAULT and let the user sort it out. + */ + free_imap(imp); + return -EFAULT ; + } + + + /* + * Build scatterlist, one entry per page + */ + i = 1; + imp->sg[0].page = imp->pages[0]; + imp->sg[0].offset = ((unsigned long)m.virtaddr) & (PAGE_SIZE - 1); + imp->sg[0].length = PAGE_SIZE - imp->sg[0].offset; + if (imp->sg[0].length >= m.size) { + imp->sg[0].length = m.size; + } else { + unsigned long len = m.size - imp->sg[0].length; + for (;len >= PAGE_SIZE && i < npages ; i++) { + imp->sg[i].page = imp->pages[i]; + imp->sg[i].offset = 0; + imp->sg[i].length = PAGE_SIZE; + len -= PAGE_SIZE; + } + if (len) { + BUG_ON(i >= npages); + BUG_ON(len >= PAGE_SIZE); + imp->sg[i].page = imp->pages[i]; + imp->sg[i].offset = 0; + imp->sg[i].length = len; + i++; + } + } + + imp->nent = i; + + m.nents = pci_map_sg(devp, imp->sg, imp->nent, m.direction); + + hw = get_hw(m.nents); + + for (i = 0, sgp = imp->sg; i < m.nents; i++, sgp++) { + hw[i].dmaaddr = sg_dma_address(sgp); + hw[i].len = sg_dma_len(sgp); + } + + imp->m.dmaaddr = hw[0].dmaaddr; + + if (copy_to_user(mp, &m, sizeof m) || + copy_to_user(m.sglist, hw, sizeof(*hw) * m.nents)) { + pci_unmap_sg(devp, imp->sg, imp->nent, imp->m.direction); + free_imap(imp); + return -EFAULT; + } + + free_hw(m.nents, hw); + + spin_lock_irqsave(&p->lk, flags); + imp->next = p->mappings; + p->mappings = imp; + spin_unlock_irqrestore(&p->lk, flags); + + return 0; + } + case USR_UNMAP: + { + struct internal_mapping_info **ipp; + imp = NULL; + spin_lock_irqsave(&p->lk, flags); + for (ipp = &p->mappings; *ipp; ipp = &(*ipp)->next) { + imp = *ipp; + if (imp->m.dmaaddr == m.dmaaddr) { + *ipp = imp->next; + break; + } + } + spin_unlock_irqrestore(&p->lk, flags); + + if (imp == NULL) + return -EINVAL; + + + pci_unmap_sg(devp, imp->sg, imp->nent, imp->m.direction); + + free_imap(imp); + + return 0; + } + default: + return -EINVAL; + } +} + +static struct inode * +usr_pci_inode(void) +{ + struct inode *inode = new_inode(usr_pci_mnt->mnt_sb); + if (!inode) + return (struct inode *)-ENOMEM; + + inode->i_fop = &usr_pci_fops; + + /* + * Mark the inode dirty from the very beginning, + * that way it will never be moved to the dirty + * list because mark_inode_dirty() will think + * that it already _is_ on the dirty list. + */ + inode->i_state = I_DIRTY; + inode->i_mode = S_IRUSR | S_IWUSR; + inode->i_uid = current->fsuid; + inode->i_gid = current->fsgid; + inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME; + inode->i_blksize = PAGE_SIZE; + return inode; +} + +int +usr_pci_get_dev(int fd, struct pci_dev **pp, struct file **fpp) +{ + struct file *filp = fget(fd); + struct maplist *p; + + if (!filp) + return -EINVAL; + if (filp->f_op != &usr_pci_fops) { + fput(filp); + return -EINVAL; + } + p = (struct maplist *)filp->private_data; + + if (!p) { + printk(KERN_ERR "usr_pci_fs file has no PCI device???"); + return -EINVAL; + } + *pp = p->dev; + *fpp = filp; + return 0; +} + +void +usr_pci_put_dev(struct file *fp) +{ + fput(fp); +} +EXPORT_SYMBOL(usr_pci_get_dev); +EXPORT_SYMBOL(usr_pci_put_dev); + +int asmlinkage +usr_pci_open_sys(int bus, int slot, int function) +{ + struct qstr this; + int error; + int fn; + + char name[32]; + struct pci_dev *devp = NULL; + struct inode *inode; + struct dentry *dentry; + struct file *file; + struct maplist *mp; + int fd; + + + fn = ((slot & 0x1f) << 3) | (function & 0x7); + + mp = kmalloc(sizeof *mp, GFP_KERNEL); + if (mp == NULL) + return -ENOMEM; + mp->mappings = NULL; + spin_lock_init(&mp->lk); + + devp = pci_find_slot(bus, fn); + if (devp == NULL) { + error = -ENOENT; + goto out1; + } + + mp->dev = devp; + + this.hash = (bus<<8) | fn; + sprintf(name, "%d.%x.%d", bus, slot, fn); + this.len = strlen(name); + this.name = name; + + /* + * RACE here -- another process could grab the device after + * we've checked. Also we don't check properly against + * in-kernel device uses. + */ + dentry = d_lookup(usr_pci_mnt->mnt_sb->s_root, &this); + if (dentry) { + error = -EBUSY; + goto out2; + } + + dentry = d_alloc(usr_pci_mnt->mnt_sb->s_root, &this); + if (!dentry) { + error = -ENOMEM; + goto out1; + } + + if ((error = pci_enable_device(devp)) != 0) { + goto out2; + } + + pci_set_master(devp); + + + file = get_empty_filp(); + if (!file) { + error = -ENFILE; + goto out3; + } + + inode = usr_pci_inode(); + if (IS_ERR(inode)) { + error = PTR_ERR(inode); + goto out4; + } + + dentry->d_op = &usr_pci_dentry_operations; + d_add(dentry, inode); + + fd = error = get_unused_fd(); + if (error < 0) { + goto out5; + } + + file->f_vfsmnt = mntget(usr_pci_mnt); + + /* + * No dget() here -- we want this entry to go away when the + * file is closed. + */ + file->f_dentry = dentry; + file->f_op = &usr_pci_fops; + file->private_data = (void *)mp; + + fd_install(fd, file); + return fd; + + out5: + iput(inode); + out4: + put_filp(file); + out3: + pci_disable_device(devp); + out2: + dput(dentry); + out1: + kfree(mp); + return error; +} + +/* + * Filesystem glue + */ +int __init +usr_pci_init(void) +{ + int error; + + error = register_filesystem(&usr_pci_fs_type); + if (error) + return error; + usr_pci_mnt = kern_mount(&usr_pci_fs_type); + if (IS_ERR(usr_pci_mnt)) { + unregister_filesystem(&usr_pci_fs_type); + return PTR_ERR(usr_pci_mnt); + } + + return 0; +} + +static int +usr_pci_release(struct inode *ip, struct file *fp) +{ + struct maplist *mp = (struct maplist *)fp->private_data; + struct internal_mapping_info *mip; + unsigned long flags; + + spin_lock_irqsave(&mp->lk, flags); + mip = mp->mappings; + mp->mappings = NULL; + spin_unlock_irqrestore(&mp->lk, flags); + + while (mip) { + struct internal_mapping_info *im = mip; + mip = mip->next; + + pci_unmap_sg(mp->dev, im->sg, im->nent, im->m.direction); + free_imap(im); + } + + + pci_disable_device(mp->dev); + fp->private_data = NULL; + kfree(mp); + return 0; +} + + +__initcall(usr_pci_init); + +#else /* CONFIG_USRDEV */ + +int asmlinkage +usr_pci_open_sys(int bus, int slot, int function) +{ + return -ENOSYS; +} + +int asmlinkage +usr_pci_map_sys(int fd, int action, struct mapping_info *mp) +{ + return -ENOSYS; +} +#endif /* CONFIG_USRDEV */ + diff -Nur --exclude=RCS --exclude=CVS --exclude=SCCS --exclude=BitKeeper --exclude=ChangeSet linux-2.5-import/include/asm-alpha/unistd.h linux-2.5-usrdrivers/include/asm-alpha/unistd.h --- linux-2.5-import/include/asm-alpha/unistd.h Tue Jul 15 08:48:02 2003 +++ linux-2.5-usrdrivers/include/asm-alpha/unistd.h Wed Nov 5 11:05:02 2003 @@ -360,7 +360,9 @@ #define __NR_clock_nanosleep 422 #define __NR_semtimedop 423 #define __NR_tgkill 424 -#define NR_SYSCALLS 425 +#define __NR_usr_pci_map 425 +#define __NR_usr_pci_open 426 +#define NR_SYSCALLS 427 #if defined(__GNUC__) diff -Nur --exclude=RCS --exclude=CVS --exclude=SCCS --exclude=BitKeeper --exclude=ChangeSet linux-2.5-import/include/asm-i386/unistd.h linux-2.5-usrdrivers/include/asm-i386/unistd.h --- linux-2.5-import/include/asm-i386/unistd.h Fri Oct 3 08:26:49 2003 +++ linux-2.5-usrdrivers/include/asm-i386/unistd.h Wed Nov 5 11:05:02 2003 @@ -279,8 +279,10 @@ #define __NR_utimes 271 #define __NR_fadvise64_64 272 #define __NR_vserver 273 +#define __NR_usr_pci_map 274 +#define __NR_usr_pci_open 275 -#define NR_syscalls 274 +#define NR_syscalls 276 /* user-visible error numbers are in the range -1 - -124: see */ diff -Nur --exclude=RCS --exclude=CVS --exclude=SCCS --exclude=BitKeeper --exclude=ChangeSet linux-2.5-import/include/asm-ia64/unistd.h linux-2.5-usrdrivers/include/asm-ia64/unistd.h --- linux-2.5-import/include/asm-ia64/unistd.h Wed Oct 15 15:30:03 2003 +++ linux-2.5-usrdrivers/include/asm-ia64/unistd.h Wed Nov 5 11:05:03 2003 @@ -248,10 +248,11 @@ #define __NR_clock_nanosleep 1256 #define __NR_fstatfs64 1257 #define __NR_statfs64 1258 - +#define __NR_usr_pci_map 1259 +#define __NR_usr_pci_open 1260 #ifdef __KERNEL__ -#define NR_syscalls 256 /* length of syscall table */ +#define NR_syscalls 275 /* length of syscall table */ #if !defined(__ASSEMBLY__) && !defined(ASSEMBLER) diff -Nur --exclude=RCS --exclude=CVS --exclude=SCCS --exclude=BitKeeper --exclude=ChangeSet linux-2.5-import/include/linux/usrblk.h linux-2.5-usrdrivers/include/linux/usrblk.h --- linux-2.5-import/include/linux/usrblk.h Thu Jan 1 10:00:00 1970 +++ linux-2.5-usrdrivers/include/linux/usrblk.h Wed Nov 5 11:05:03 2003 @@ -0,0 +1,60 @@ +/************************************************************************ + * usrblk.h -- definitions for user-mode block devices. + */ + +#ifndef _USRBLK_H +#define _USRBLK_H + +#include +#include + +enum usrblk_cmd { + blk_read = 0, + blk_write, + blk_identify, + blk_io_complete, + blk_io_pci, +}; + + +/* + * Things read from the FD + */ +struct usr_io_req { + __u8 cmd; /* values from enum blk_cmd */ + __u8 nentries; /* Number of valid entries in scatterlist */ + __u8 unit; /* which disc (channel * 2 + drive id) */ + __u8 raw_nentries; /* size of scatterlist */ + __u32 nsectors; /* number of sectors to transfer */ + __u64 tag; /* random ID for IOcomplete */ + __u64 startsector; /* where to start reading/writing */ + struct scatterlist sglist[]; +}; + + + +/* + * Things written from userspace to the FD. + */ +struct usr_io_complete { + enum usrblk_cmd what; + union { + struct blk_identify { /* what == identify */ + __u8 unit; /* channel * 2 + drive ID */ + __u16 sectsize; /* in bytes */ + __u64 capacity; /* in sectors */ + + } ident; + struct blk_iocomplete { /* what == blk_io_complete */ + __u64 tag; + __u64 sector; /* error sector for errcode != 0 */ + __u32 errcode; + } err; + struct blk_pci { + __u32 fd; /* file descriptor from usr_pci_open() */ + } pcifd; + } u; +}; + +#endif + diff -Nur --exclude=RCS --exclude=CVS --exclude=SCCS --exclude=BitKeeper --exclude=ChangeSet linux-2.5-import/include/linux/usrdrv.h linux-2.5-usrdrivers/include/linux/usrdrv.h --- linux-2.5-import/include/linux/usrdrv.h Thu Jan 1 10:00:00 1970 +++ linux-2.5-usrdrivers/include/linux/usrdrv.h Wed Nov 5 11:05:03 2003 @@ -0,0 +1,84 @@ +/************************************************************************ + * usrdrv.h -- definitions for user-mode device driver + */ + +#ifndef _USRDRV_H +#define _USRDRV_H +#ifdef __KERNEL__ +#include + +extern void * +usrdev_do_map(void *kaddr, + unsigned long origlen, + unsigned long dmaaddr, + struct pci_dev *devp); + + +struct file; +struct pci_dev; +extern void usr_pci_put_dev(struct file *fp); +extern int usr_pci_get_dev(int, struct pci_dev **, struct file **); + +#else +/* + * can't include linux/dma-mapping.h in user mode + */ +enum dma_data_direction { + DMA_BIDIRECTIONAL = 0, + DMA_TO_DEVICE = 1, + DMA_FROM_DEVICE = 2, + DMA_NONE = 3, +}; +#endif + +struct usr_pci_sglist { + unsigned long dmaaddr; + unsigned long len; +}; + + +/* + * virtaddr: user mode address to be mapped/unmapped + * size: bytes of address to map + * nents: As passed into usr_pci_map will contain total + * number of entries; + * as passed out, will contain number of valid entries + * (IOMMU may merge entries) + * sglist: allocated by caller of usr_pci_map, + * should be at least (size/PAGE_SIZE) + 2 + * direction: try not to use DMA_BIDERECTIONAL + */ +struct mapping_info { + void *virtaddr; + unsigned long dmaaddr; + unsigned int size; + unsigned int nents; + struct usr_pci_sglist *sglist; + enum dma_data_direction direction; +}; + + +/* + * Actions for usr_pci_map + */ +#define USR_MAP 1 +#define USR_UNMAP 2 +#define USR_SYNC 3 +#define USR_ALLOC_CONSISTENT 4 + + +/* + * Map a portion of virtual memory to + * be accessible via DMA from a PCI device. + * First argument (pcifd) must be obtained from usr_pci_open(). + */ +extern int usr_pci_map(int pcifd, int action, struct mapping_info *mp); + +/* + * Enable a PCI device, and return a file descriptor to map DMA to/from it by. + * Call close() on returned value to disable device again. + */ +extern int usr_pci_open(int bus, int slot, int fn); + + +#endif /* _USRDRV_H */ diff -Nur --exclude=RCS --exclude=CVS --exclude=SCCS --exclude=BitKeeper --exclude=ChangeSet linux-2.5-import/kernel/sched.c-new linux-2.5-usrdrivers/kernel/sched.c-new --- linux-2.5-import/kernel/sched.c-new Thu Nov 6 11:38:46 2003 +++ linux-2.5-usrdrivers/kernel/sched.c-new Thu Jan 1 10:00:00 1970 @@ -1,2915 +0,0 @@ -/* - * kernel/sched.c - * - * Kernel scheduler and related syscalls - * - * Copyright (C) 1991-2002 Linus Torvalds - * - * 1996-12-23 Modified by Dave Grothe to fix bugs in semaphores and - * make semaphores SMP safe - * 1998-11-19 Implemented schedule_timeout() and related stuff - * by Andrea Arcangeli - * 2002-01-04 New ultra-scalable O(1) scheduler by Ingo Molnar: - * hybrid priority-list and round-robin design with - * an array-switch method of distributing timeslices - * and per-CPU runqueues. Cleanups and useful suggestions - * by Davide Libenzi, preemptible kernel bits by Robert Love. - * 2003-09-03 Interactivity tuning by Con Kolivas. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef CONFIG_NUMA -#define cpu_to_node_mask(cpu) node_to_cpumask(cpu_to_node(cpu)) -#else -#define cpu_to_node_mask(cpu) (cpu_online_map) -#endif - -/* - * Convert user-nice values [ -20 ... 0 ... 19 ] - * to static priority [ MAX_RT_PRIO..MAX_PRIO-1 ], - * and back. - */ -#define NICE_TO_PRIO(nice) (MAX_RT_PRIO + (nice) + 20) -#define PRIO_TO_NICE(prio) ((prio) - MAX_RT_PRIO - 20) -#define TASK_NICE(p) PRIO_TO_NICE((p)->static_prio) - -/* - * 'User priority' is the nice value converted to something we - * can work with better when scaling various scheduler parameters, - * it's a [ 0 ... 39 ] range. - */ -#define USER_PRIO(p) ((p)-MAX_RT_PRIO) -#define TASK_USER_PRIO(p) USER_PRIO((p)->static_prio) -#define MAX_USER_PRIO (USER_PRIO(MAX_PRIO)) -#define AVG_TIMESLICE (MIN_TIMESLICE + ((MAX_TIMESLICE - MIN_TIMESLICE) *\ - (MAX_PRIO-1-NICE_TO_PRIO(0))/(MAX_USER_PRIO - 1))) - -/* - * Some helpers for converting nanosecond timing to jiffy resolution - */ -#define NS_TO_JIFFIES(TIME) ((TIME) / (1000000000 / HZ)) -#define JIFFIES_TO_NS(TIME) ((TIME) * (1000000000 / HZ)) - -/* - * These are the 'tuning knobs' of the scheduler: - * - * Minimum timeslice is 10 msecs, default timeslice is 100 msecs, - * maximum timeslice is 200 msecs. Timeslices get refilled after - * they expire. - */ -#define MIN_TIMESLICE ( 10 * HZ / 1000) -#define MAX_TIMESLICE (200 * HZ / 1000) -#define ON_RUNQUEUE_WEIGHT 30 -#define CHILD_PENALTY 95 -#define PARENT_PENALTY 100 -#define EXIT_WEIGHT 3 -#define PRIO_BONUS_RATIO 25 -#define MAX_BONUS (MAX_USER_PRIO * PRIO_BONUS_RATIO / 100) -#define INTERACTIVE_DELTA 2 -#define MAX_SLEEP_AVG (AVG_TIMESLICE * MAX_BONUS) -#define STARVATION_LIMIT (MAX_SLEEP_AVG) -#define NS_MAX_SLEEP_AVG (JIFFIES_TO_NS(MAX_SLEEP_AVG)) -#define NODE_THRESHOLD 125 -#define CREDIT_LIMIT 100 - -/* - * If a task is 'interactive' then we reinsert it in the active - * array after it has expired its current timeslice. (it will not - * continue to run immediately, it will still roundrobin with - * other interactive tasks.) - * - * This part scales the interactivity limit depending on niceness. - * - * We scale it linearly, offset by the INTERACTIVE_DELTA delta. - * Here are a few examples of different nice levels: - * - * TASK_INTERACTIVE(-20): [1,1,1,1,1,1,1,1,1,0,0] - * TASK_INTERACTIVE(-10): [1,1,1,1,1,1,1,0,0,0,0] - * TASK_INTERACTIVE( 0): [1,1,1,1,0,0,0,0,0,0,0] - * TASK_INTERACTIVE( 10): [1,1,0,0,0,0,0,0,0,0,0] - * TASK_INTERACTIVE( 19): [0,0,0,0,0,0,0,0,0,0,0] - * - * (the X axis represents the possible -5 ... 0 ... +5 dynamic - * priority range a task can explore, a value of '1' means the - * task is rated interactive.) - * - * Ie. nice +19 tasks can never get 'interactive' enough to be - * reinserted into the active array. And only heavily CPU-hog nice -20 - * tasks will be expired. Default nice 0 tasks are somewhere between, - * it takes some effort for them to get interactive, but it's not - * too hard. - */ - -#define CURRENT_BONUS(p) \ - (NS_TO_JIFFIES((p)->sleep_avg) * MAX_BONUS / \ - MAX_SLEEP_AVG) - -#ifdef CONFIG_SMP -#define TIMESLICE_GRANULARITY(p) (MIN_TIMESLICE * \ - (1 << (((MAX_BONUS - CURRENT_BONUS(p)) ? : 1) - 1)) * \ - num_online_cpus()) -#else -#define TIMESLICE_GRANULARITY(p) (MIN_TIMESLICE * \ - (1 << (((MAX_BONUS - CURRENT_BONUS(p)) ? : 1) - 1))) -#endif - -#define SCALE(v1,v1_max,v2_max) \ - (v1) * (v2_max) / (v1_max) - -#define DELTA(p) \ - (SCALE(TASK_NICE(p), 40, MAX_USER_PRIO*PRIO_BONUS_RATIO/100) + \ - INTERACTIVE_DELTA) - -#define TASK_INTERACTIVE(p) \ - ((p)->prio <= (p)->static_prio - DELTA(p)) - -#define JUST_INTERACTIVE_SLEEP(p) \ - (JIFFIES_TO_NS(MAX_SLEEP_AVG * \ - (MAX_BONUS / 2 + DELTA((p)) + 1) / MAX_BONUS - 1)) - -#define HIGH_CREDIT(p) \ - ((p)->interactive_credit > CREDIT_LIMIT) - -#define LOW_CREDIT(p) \ - ((p)->interactive_credit < -CREDIT_LIMIT) - -#define TASK_PREEMPTS_CURR(p, rq) \ - ((p)->prio < (rq)->curr->prio) - -/* - * BASE_TIMESLICE scales user-nice values [ -20 ... 19 ] - * to time slice values. - * - * The higher a thread's priority, the bigger timeslices - * it gets during one round of execution. But even the lowest - * priority thread gets MIN_TIMESLICE worth of execution time. - * - * task_timeslice() is the interface that is used by the scheduler. - */ - -#define BASE_TIMESLICE(p) (MIN_TIMESLICE + \ - ((MAX_TIMESLICE - MIN_TIMESLICE) * (MAX_PRIO-1-(p)->static_prio)/(MAX_USER_PRIO - 1))) - -static inline unsigned int task_timeslice(task_t *p) -{ - return BASE_TIMESLICE(p); -} - -/* - * These are the runqueue data structures: - */ - -#define BITMAP_SIZE ((((MAX_PRIO+1+7)/8)+sizeof(long)-1)/sizeof(long)) - -typedef struct runqueue runqueue_t; - -struct prio_array { - int nr_active; - unsigned long bitmap[BITMAP_SIZE]; - struct list_head queue[MAX_PRIO]; -}; - -/* - * This is the main, per-CPU runqueue data structure. - * - * Locking rule: those places that want to lock multiple runqueues - * (such as the load balancing or the thread migration code), lock - * acquire operations must be ordered by ascending &runqueue. - */ -struct runqueue { - spinlock_t lock; - unsigned long nr_running, nr_switches, expired_timestamp, - nr_uninterruptible; - task_t *curr, *idle; - struct mm_struct *prev_mm; - prio_array_t *active, *expired, arrays[2]; - int prev_cpu_load[NR_CPUS]; -#ifdef CONFIG_NUMA - atomic_t *node_nr_running; - int prev_node_load[MAX_NUMNODES]; -#endif - task_t *migration_thread; - struct list_head migration_queue; - - atomic_t nr_iowait; -}; - -static DEFINE_PER_CPU(struct runqueue, runqueues); - -#define cpu_rq(cpu) (&per_cpu(runqueues, (cpu))) -#define this_rq() (&__get_cpu_var(runqueues)) -#define task_rq(p) cpu_rq(task_cpu(p)) -#define cpu_curr(cpu) (cpu_rq(cpu)->curr) - -/* - * Default context-switch locking: - */ -#ifndef prepare_arch_switch -# define prepare_arch_switch(rq, next) do { } while(0) -# define finish_arch_switch(rq, next) spin_unlock_irq(&(rq)->lock) -# define task_running(rq, p) ((rq)->curr == (p)) -#endif - -#ifdef CONFIG_NUMA - -/* - * Keep track of running tasks. - */ - -static atomic_t node_nr_running[MAX_NUMNODES] ____cacheline_maxaligned_in_smp = - {[0 ...MAX_NUMNODES-1] = ATOMIC_INIT(0)}; - -static inline void nr_running_init(struct runqueue *rq) -{ - rq->node_nr_running = &node_nr_running[0]; -} - -static inline void nr_running_inc(runqueue_t *rq) -{ - atomic_inc(rq->node_nr_running); - rq->nr_running++; -} - -static inline void nr_running_dec(runqueue_t *rq) -{ - atomic_dec(rq->node_nr_running); - rq->nr_running--; -} - -__init void node_nr_running_init(void) -{ - int i; - - for (i = 0; i < NR_CPUS; i++) { - if (cpu_possible(i)) - cpu_rq(i)->node_nr_running = - &node_nr_running[cpu_to_node(i)]; - } -} - -#else /* !CONFIG_NUMA */ - -# define nr_running_init(rq) do { } while (0) -# define nr_running_inc(rq) do { (rq)->nr_running++; } while (0) -# define nr_running_dec(rq) do { (rq)->nr_running--; } while (0) - -#endif /* CONFIG_NUMA */ - -/* - * task_rq_lock - lock the runqueue a given task resides on and disable - * interrupts. Note the ordering: we can safely lookup the task_rq without - * explicitly disabling preemption. - */ -static inline runqueue_t *task_rq_lock(task_t *p, unsigned long *flags) -{ - struct runqueue *rq; - -repeat_lock_task: - local_irq_save(*flags); - rq = task_rq(p); - spin_lock(&rq->lock); - if (unlikely(rq != task_rq(p))) { - spin_unlock_irqrestore(&rq->lock, *flags); - goto repeat_lock_task; - } - return rq; -} - -static inline void task_rq_unlock(runqueue_t *rq, unsigned long *flags) -{ - spin_unlock_irqrestore(&rq->lock, *flags); -} - -/* - * rq_lock - lock a given runqueue and disable interrupts. - */ -static inline runqueue_t *this_rq_lock(void) -{ - runqueue_t *rq; - - local_irq_disable(); - rq = this_rq(); - spin_lock(&rq->lock); - - return rq; -} - -static inline void rq_unlock(runqueue_t *rq) -{ - spin_unlock_irq(&rq->lock); -} - -/* - * Adding/removing a task to/from a priority array: - */ -static inline void dequeue_task(struct task_struct *p, prio_array_t *array) -{ - array->nr_active--; - list_del(&p->run_list); - if (list_empty(array->queue + p->prio)) - __clear_bit(p->prio, array->bitmap); -} - -static inline void enqueue_task(struct task_struct *p, prio_array_t *array) -{ - list_add_tail(&p->run_list, array->queue + p->prio); - __set_bit(p->prio, array->bitmap); - array->nr_active++; - p->array = array; -} - -/* - * effective_prio - return the priority that is based on the static - * priority but is modified by bonuses/penalties. - * - * We scale the actual sleep average [0 .... MAX_SLEEP_AVG] - * into the -5 ... 0 ... +5 bonus/penalty range. - * - * We use 25% of the full 0...39 priority range so that: - * - * 1) nice +19 interactive tasks do not preempt nice 0 CPU hogs. - * 2) nice -20 CPU hogs do not get preempted by nice 0 tasks. - * - * Both properties are important to certain workloads. - */ -static int effective_prio(task_t *p) -{ - int bonus, prio; - - if (rt_task(p)) - return p->prio; - - bonus = CURRENT_BONUS(p) - MAX_BONUS / 2; - - prio = p->static_prio - bonus; - if (prio < MAX_RT_PRIO) - prio = MAX_RT_PRIO; - if (prio > MAX_PRIO-1) - prio = MAX_PRIO-1; - return prio; -} - -/* - * __activate_task - move a task to the runqueue. - */ -static inline void __activate_task(task_t *p, runqueue_t *rq) -{ - enqueue_task(p, rq->active); - nr_running_inc(rq); -} - -static void recalc_task_prio(task_t *p, unsigned long long now) -{ - unsigned long long __sleep_time = now - p->timestamp; - unsigned long sleep_time; - - if (__sleep_time > NS_MAX_SLEEP_AVG) - sleep_time = NS_MAX_SLEEP_AVG; - else - sleep_time = (unsigned long)__sleep_time; - - if (likely(sleep_time > 0)) { - /* - * User tasks that sleep a long time are categorised as - * idle and will get just interactive status to stay active & - * prevent them suddenly becoming cpu hogs and starving - * other processes. - */ - if (p->mm && p->activated != -1 && - sleep_time > JUST_INTERACTIVE_SLEEP(p)){ - p->sleep_avg = JIFFIES_TO_NS(MAX_SLEEP_AVG - - AVG_TIMESLICE); - if (!HIGH_CREDIT(p)) - p->interactive_credit++; - } else { - /* - * The lower the sleep avg a task has the more - * rapidly it will rise with sleep time. - */ - sleep_time *= (MAX_BONUS - CURRENT_BONUS(p)) ? : 1; - - /* - * Tasks with low interactive_credit are limited to - * one timeslice worth of sleep avg bonus. - */ - if (LOW_CREDIT(p) && - sleep_time > JIFFIES_TO_NS(task_timeslice(p))) - sleep_time = - JIFFIES_TO_NS(task_timeslice(p)); - - /* - * Non high_credit tasks waking from uninterruptible - * sleep are limited in their sleep_avg rise as they - * are likely to be cpu hogs waiting on I/O - */ - if (p->activated == -1 && !HIGH_CREDIT(p) && p->mm){ - if (p->sleep_avg >= JUST_INTERACTIVE_SLEEP(p)) - sleep_time = 0; - else if (p->sleep_avg + sleep_time >= - JUST_INTERACTIVE_SLEEP(p)){ - p->sleep_avg = - JUST_INTERACTIVE_SLEEP(p); - sleep_time = 0; - } - } - - /* - * This code gives a bonus to interactive tasks. - * - * The boost works by updating the 'average sleep time' - * value here, based on ->timestamp. The more time a task - * spends sleeping, the higher the average gets - and the - * higher the priority boost gets as well. - */ - p->sleep_avg += sleep_time; - - if (p->sleep_avg > NS_MAX_SLEEP_AVG){ - p->sleep_avg = NS_MAX_SLEEP_AVG; - if (!HIGH_CREDIT(p)) - p->interactive_credit++; - } - } - } - - p->prio = effective_prio(p); -} - -/* - * activate_task - move a task to the runqueue and do priority recalculation - * - * Update all the scheduling statistics stuff. (sleep average - * calculation, priority modifiers, etc.) - */ -static inline void activate_task(task_t *p, runqueue_t *rq) -{ - unsigned long long now = sched_clock(); - - recalc_task_prio(p, now); - - /* - * This checks to make sure it's not an uninterruptible task - * that is now waking up. - */ - if (!p->activated){ - /* - * Tasks which were woken up by interrupts (ie. hw events) - * are most likely of interactive nature. So we give them - * the credit of extending their sleep time to the period - * of time they spend on the runqueue, waiting for execution - * on a CPU, first time around: - */ - if (in_interrupt()) - p->activated = 2; - else - /* - * Normal first-time wakeups get a credit too for on-runqueue - * time, but it will be weighted down: - */ - p->activated = 1; - } - p->timestamp = now; - - __activate_task(p, rq); -} - -/* - * deactivate_task - remove a task from the runqueue. - */ -static inline void deactivate_task(struct task_struct *p, runqueue_t *rq) -{ - nr_running_dec(rq); - if (p->state == TASK_UNINTERRUPTIBLE) - rq->nr_uninterruptible++; - dequeue_task(p, p->array); - p->array = NULL; -} - -/* - * resched_task - mark a task 'to be rescheduled now'. - * - * On UP this means the setting of the need_resched flag, on SMP it - * might also involve a cross-CPU call to trigger the scheduler on - * the target CPU. - */ -static inline void resched_task(task_t *p) -{ -#ifdef CONFIG_SMP - int need_resched, nrpolling; - - preempt_disable(); - /* minimise the chance of sending an interrupt to poll_idle() */ - nrpolling = test_tsk_thread_flag(p,TIF_POLLING_NRFLAG); - need_resched = test_and_set_tsk_thread_flag(p,TIF_NEED_RESCHED); - nrpolling |= test_tsk_thread_flag(p,TIF_POLLING_NRFLAG); - - if (!need_resched && !nrpolling && (task_cpu(p) != smp_processor_id())) - smp_send_reschedule(task_cpu(p)); - preempt_enable(); -#else - set_tsk_need_resched(p); -#endif -} - -#ifdef CONFIG_SMP - -/* - * wait_task_inactive - wait for a thread to unschedule. - * - * The caller must ensure that the task *will* unschedule sometime soon, - * else this function might spin for a *long* time. This function can't - * be called with interrupts off, or it may introduce deadlock with - * smp_call_function() if an IPI is sent by the same process we are - * waiting to become inactive. - */ -void wait_task_inactive(task_t * p) -{ - unsigned long flags; - runqueue_t *rq; - -repeat: - preempt_disable(); - rq = task_rq(p); - if (unlikely(task_running(rq, p))) { - cpu_relax(); - /* - * enable/disable preemption just to make this - * a preemption point - we are busy-waiting - * anyway. - */ - preempt_enable(); - goto repeat; - } - rq = task_rq_lock(p, &flags); - if (unlikely(task_running(rq, p))) { - task_rq_unlock(rq, &flags); - preempt_enable(); - goto repeat; - } - task_rq_unlock(rq, &flags); - preempt_enable(); -} -#endif - -/*** - * try_to_wake_up - wake up a thread - * @p: the to-be-woken-up thread - * @state: the mask of task states that can be woken - * @sync: do a synchronous wakeup? - * @kick: kick the CPU if the task is already running? - * - * Put it on the run-queue if it's not already there. The "current" - * thread is always on the run-queue (except when the actual - * re-schedule is in progress), and as such you're allowed to do - * the simpler "current->state = TASK_RUNNING" to mark yourself - * runnable without the overhead of this. - * - * returns failure only if the task is already active. - */ -static int try_to_wake_up(task_t * p, unsigned int state, int sync, int kick) -{ - unsigned long flags; - int success = 0; - long old_state; - runqueue_t *rq; - -repeat_lock_task: - rq = task_rq_lock(p, &flags); - old_state = p->state; - if (old_state & state) { - if (!p->array) { - /* - * Fast-migrate the task if it's not running or runnable - * currently. Do not violate hard affinity. - */ - if (unlikely(sync && !task_running(rq, p) && - (task_cpu(p) != smp_processor_id()) && - cpu_isset(smp_processor_id(), p->cpus_allowed))) { - - set_task_cpu(p, smp_processor_id()); - task_rq_unlock(rq, &flags); - goto repeat_lock_task; - } - if (old_state == TASK_UNINTERRUPTIBLE){ - rq->nr_uninterruptible--; - /* - * Tasks on involuntary sleep don't earn - * sleep_avg beyond just interactive state. - */ - p->activated = -1; - } - if (sync) - __activate_task(p, rq); - else { - activate_task(p, rq); - if (TASK_PREEMPTS_CURR(p, rq)) - resched_task(rq->curr); - } - success = 1; - } -#ifdef CONFIG_SMP - else - if (unlikely(kick) && task_running(rq, p) && (task_cpu(p) != smp_processor_id())) - smp_send_reschedule(task_cpu(p)); -#endif - p->state = TASK_RUNNING; - } - task_rq_unlock(rq, &flags); - - return success; -} - -int wake_up_process(task_t * p) -{ - return try_to_wake_up(p, TASK_STOPPED | TASK_INTERRUPTIBLE | TASK_UNINTERRUPTIBLE, 0, 0); -} - -EXPORT_SYMBOL(wake_up_process); - -int wake_up_process_kick(task_t * p) -{ - return try_to_wake_up(p, TASK_STOPPED | TASK_INTERRUPTIBLE | TASK_UNINTERRUPTIBLE, 0, 1); -} - -int wake_up_state(task_t *p, unsigned int state) -{ - return try_to_wake_up(p, state, 0, 0); -} - -/* - * wake_up_forked_process - wake up a freshly forked process. - * - * This function will do some initial scheduler statistics housekeeping - * that must be done for every newly created process. - */ -void wake_up_forked_process(task_t * p) -{ - unsigned long flags; - runqueue_t *rq = task_rq_lock(current, &flags); - - p->state = TASK_RUNNING; - /* - * We decrease the sleep average of forking parents - * and children as well, to keep max-interactive tasks - * from forking tasks that are max-interactive. - */ - current->sleep_avg = JIFFIES_TO_NS(CURRENT_BONUS(current) * - PARENT_PENALTY / 100 * MAX_SLEEP_AVG / MAX_BONUS); - - p->sleep_avg = JIFFIES_TO_NS(CURRENT_BONUS(p) * - CHILD_PENALTY / 100 * MAX_SLEEP_AVG / MAX_BONUS); - - p->interactive_credit = 0; - - p->prio = effective_prio(p); - set_task_cpu(p, smp_processor_id()); - - if (unlikely(!current->array)) - __activate_task(p, rq); - else { - p->prio = current->prio; - list_add_tail(&p->run_list, ¤t->run_list); - p->array = current->array; - p->array->nr_active++; - nr_running_inc(rq); - } - task_rq_unlock(rq, &flags); -} - -/* - * Potentially available exiting-child timeslices are - * retrieved here - this way the parent does not get - * penalized for creating too many threads. - * - * (this cannot be used to 'generate' timeslices - * artificially, because any timeslice recovered here - * was given away by the parent in the first place.) - */ -void sched_exit(task_t * p) -{ - unsigned long flags; - - local_irq_save(flags); - if (p->first_time_slice) { - p->parent->time_slice += p->time_slice; - if (unlikely(p->parent->time_slice > MAX_TIMESLICE)) - p->parent->time_slice = MAX_TIMESLICE; - } - local_irq_restore(flags); - /* - * If the child was a (relative-) CPU hog then decrease - * the sleep_avg of the parent as well. - */ - if (p->sleep_avg < p->parent->sleep_avg) - p->parent->sleep_avg = p->parent->sleep_avg / - (EXIT_WEIGHT + 1) * EXIT_WEIGHT + p->sleep_avg / - (EXIT_WEIGHT + 1); -} - -/** - * finish_task_switch - clean up after a task-switch - * @prev: the thread we just switched away from. - * - * We enter this with the runqueue still locked, and finish_arch_switch() - * will unlock it along with doing any other architecture-specific cleanup - * actions. - * - * Note that we may have delayed dropping an mm in context_switch(). If - * so, we finish that here outside of the runqueue lock. (Doing it - * with the lock held can cause deadlocks; see schedule() for - * details.) - */ -static inline void finish_task_switch(task_t *prev) -{ - runqueue_t *rq = this_rq(); - struct mm_struct *mm = rq->prev_mm; - unsigned long prev_task_flags; - - rq->prev_mm = NULL; - - /* - * A task struct has one reference for the use as "current". - * If a task dies, then it sets TASK_ZOMBIE in tsk->state and calls - * schedule one last time. The schedule call will never return, - * and the scheduled task must drop that reference. - * The test for TASK_ZOMBIE must occur while the runqueue locks are - * still held, otherwise prev could be scheduled on another cpu, die - * there before we look at prev->state, and then the reference would - * be dropped twice. - * Manfred Spraul - */ - prev_task_flags = prev->flags; - finish_arch_switch(rq, prev); - if (mm) - mmdrop(mm); - if (unlikely(prev_task_flags & PF_DEAD)) - put_task_struct(prev); -} - -/** - * schedule_tail - first thing a freshly forked thread must call. - * @prev: the thread we just switched away from. - */ -asmlinkage void schedule_tail(task_t *prev) -{ - finish_task_switch(prev); - - if (current->set_child_tid) - put_user(current->pid, current->set_child_tid); -} - -/* - * context_switch - switch to the new MM and the new - * thread's register state. - */ -static inline task_t * context_switch(runqueue_t *rq, task_t *prev, task_t *next) -{ - struct mm_struct *mm = next->mm; - struct mm_struct *oldmm = prev->active_mm; - - if (unlikely(!mm)) { - next->active_mm = oldmm; - atomic_inc(&oldmm->mm_count); - enter_lazy_tlb(oldmm, next); - } else - switch_mm(oldmm, mm, next); - - if (unlikely(!prev->mm)) { - prev->active_mm = NULL; - WARN_ON(rq->prev_mm); - rq->prev_mm = oldmm; - } - - /* Here we just switch the register state and the stack. */ - switch_to(prev, next, prev); - - return prev; -} - -/* - * nr_running, nr_uninterruptible and nr_context_switches: - * - * externally visible scheduler statistics: current number of runnable - * threads, current number of uninterruptible-sleeping threads, total - * number of context switches performed since bootup. - */ -unsigned long nr_running(void) -{ - unsigned long i, sum = 0; - - for (i = 0; i < NR_CPUS; i++) - sum += cpu_rq(i)->nr_running; - - return sum; -} - -unsigned long nr_uninterruptible(void) -{ - unsigned long i, sum = 0; - - for (i = 0; i < NR_CPUS; i++) { - if (!cpu_online(i)) - continue; - sum += cpu_rq(i)->nr_uninterruptible; - } - return sum; -} - -unsigned long nr_context_switches(void) -{ - unsigned long i, sum = 0; - - for (i = 0; i < NR_CPUS; i++) { - if (!cpu_online(i)) - continue; - sum += cpu_rq(i)->nr_switches; - } - return sum; -} - -unsigned long nr_iowait(void) -{ - unsigned long i, sum = 0; - - for (i = 0; i < NR_CPUS; ++i) { - if (!cpu_online(i)) - continue; - sum += atomic_read(&cpu_rq(i)->nr_iowait); - } - return sum; -} - -/* - * double_rq_lock - safely lock two runqueues - * - * Note this does not disable interrupts like task_rq_lock, - * you need to do so manually before calling. - */ -static inline void double_rq_lock(runqueue_t *rq1, runqueue_t *rq2) -{ - if (rq1 == rq2) - spin_lock(&rq1->lock); - else { - if (rq1 < rq2) { - spin_lock(&rq1->lock); - spin_lock(&rq2->lock); - } else { - spin_lock(&rq2->lock); - spin_lock(&rq1->lock); - } - } -} - -/* - * double_rq_unlock - safely unlock two runqueues - * - * Note this does not restore interrupts like task_rq_unlock, - * you need to do so manually after calling. - */ -static inline void double_rq_unlock(runqueue_t *rq1, runqueue_t *rq2) -{ - spin_unlock(&rq1->lock); - if (rq1 != rq2) - spin_unlock(&rq2->lock); -} - -#ifdef CONFIG_NUMA -/* - * If dest_cpu is allowed for this process, migrate the task to it. - * This is accomplished by forcing the cpu_allowed mask to only - * allow dest_cpu, which will force the cpu onto dest_cpu. Then - * the cpu_allowed mask is restored. - */ -static void sched_migrate_task(task_t *p, int dest_cpu) -{ - cpumask_t old_mask; - - old_mask = p->cpus_allowed; - if (!cpu_isset(dest_cpu, old_mask)) - return; - /* force the process onto the specified CPU */ - set_cpus_allowed(p, cpumask_of_cpu(dest_cpu)); - - /* restore the cpus allowed mask */ - set_cpus_allowed(p, old_mask); -} - -/* - * Find the least loaded CPU. Slightly favor the current CPU by - * setting its runqueue length as the minimum to start. - */ -static int sched_best_cpu(struct task_struct *p) -{ - int i, minload, load, best_cpu, node = 0; - cpumask_t cpumask; - - best_cpu = task_cpu(p); - if (cpu_rq(best_cpu)->nr_running <= 2) - return best_cpu; - - minload = 10000000; - for_each_node_with_cpus(i) { - /* - * Node load is always divided by nr_cpus_node to normalise - * load values in case cpu count differs from node to node. - * We first multiply node_nr_running by 10 to get a little - * better resolution. - */ - load = 10 * atomic_read(&node_nr_running[i]) / nr_cpus_node(i); - if (load < minload) { - minload = load; - node = i; - } - } - - minload = 10000000; - cpumask = node_to_cpumask(node); - for (i = 0; i < NR_CPUS; ++i) { - if (!cpu_isset(i, cpumask)) - continue; - if (cpu_rq(i)->nr_running < minload) { - best_cpu = i; - minload = cpu_rq(i)->nr_running; - } - } - return best_cpu; -} - -void sched_balance_exec(void) -{ - int new_cpu; - - if (numnodes > 1) { - new_cpu = sched_best_cpu(current); - if (new_cpu != smp_processor_id()) - sched_migrate_task(current, new_cpu); - } -} - -/* - * Find the busiest node. All previous node loads contribute with a - * geometrically deccaying weight to the load measure: - * load_{t} = load_{t-1}/2 + nr_node_running_{t} - * This way sudden load peaks are flattened out a bit. - * Node load is divided by nr_cpus_node() in order to compare nodes - * of different cpu count but also [first] multiplied by 10 to - * provide better resolution. - */ -static int find_busiest_node(int this_node) -{ - int i, node = -1, load, this_load, maxload; - - if (!nr_cpus_node(this_node)) - return node; - this_load = maxload = (this_rq()->prev_node_load[this_node] >> 1) - + (10 * atomic_read(&node_nr_running[this_node]) - / nr_cpus_node(this_node)); - this_rq()->prev_node_load[this_node] = this_load; - for_each_node_with_cpus(i) { - if (i == this_node) - continue; - load = (this_rq()->prev_node_load[i] >> 1) - + (10 * atomic_read(&node_nr_running[i]) - / nr_cpus_node(i)); - this_rq()->prev_node_load[i] = load; - if (load > maxload && (100*load > NODE_THRESHOLD*this_load)) { - maxload = load; - node = i; - } - } - return node; -} - -#endif /* CONFIG_NUMA */ - -#ifdef CONFIG_SMP - -/* - * double_lock_balance - lock the busiest runqueue - * - * this_rq is locked already. Recalculate nr_running if we have to - * drop the runqueue lock. - */ -static inline unsigned int double_lock_balance(runqueue_t *this_rq, - runqueue_t *busiest, int this_cpu, int idle, unsigned int nr_running) -{ - if (unlikely(!spin_trylock(&busiest->lock))) { - if (busiest < this_rq) { - spin_unlock(&this_rq->lock); - spin_lock(&busiest->lock); - spin_lock(&this_rq->lock); - /* Need to recalculate nr_running */ - if (idle || (this_rq->nr_running > this_rq->prev_cpu_load[this_cpu])) - nr_running = this_rq->nr_running; - else - nr_running = this_rq->prev_cpu_load[this_cpu]; - } else - spin_lock(&busiest->lock); - } - return nr_running; -} - -/* - * find_busiest_queue - find the busiest runqueue among the cpus in cpumask. - */ -static inline runqueue_t *find_busiest_queue(runqueue_t *this_rq, int this_cpu, int idle, int *imbalance, cpumask_t cpumask) -{ - int nr_running, load, max_load, i; - runqueue_t *busiest, *rq_src; - - /* - * We search all runqueues to find the most busy one. - * We do this lockless to reduce cache-bouncing overhead, - * we re-check the 'best' source CPU later on again, with - * the lock held. - * - * We fend off statistical fluctuations in runqueue lengths by - * saving the runqueue length during the previous load-balancing - * operation and using the smaller one the current and saved lengths. - * If a runqueue is long enough for a longer amount of time then - * we recognize it and pull tasks from it. - * - * The 'current runqueue length' is a statistical maximum variable, - * for that one we take the longer one - to avoid fluctuations in - * the other direction. So for a load-balance to happen it needs - * stable long runqueue on the target CPU and stable short runqueue - * on the local runqueue. - * - * We make an exception if this CPU is about to become idle - in - * that case we are less picky about moving a task across CPUs and - * take what can be taken. - */ - if (idle || (this_rq->nr_running > this_rq->prev_cpu_load[this_cpu])) - nr_running = this_rq->nr_running; - else - nr_running = this_rq->prev_cpu_load[this_cpu]; - - busiest = NULL; - max_load = 1; - for (i = 0; i < NR_CPUS; i++) { - if (!cpu_isset(i, cpumask)) - continue; - - rq_src = cpu_rq(i); - if (idle || (rq_src->nr_running < this_rq->prev_cpu_load[i])) - load = rq_src->nr_running; - else - load = this_rq->prev_cpu_load[i]; - this_rq->prev_cpu_load[i] = rq_src->nr_running; - - if ((load > max_load) && (rq_src != this_rq)) { - busiest = rq_src; - max_load = load; - } - } - - if (likely(!busiest)) - goto out; - - *imbalance = max_load - nr_running; - - /* It needs an at least ~25% imbalance to trigger balancing. */ - if (!idle && ((*imbalance)*4 < max_load)) { - busiest = NULL; - goto out; - } - - nr_running = double_lock_balance(this_rq, busiest, this_cpu, idle, nr_running); - /* - * Make sure nothing changed since we checked the - * runqueue length. - */ - if (busiest->nr_running <= nr_running) { - spin_unlock(&busiest->lock); - busiest = NULL; - } -out: - return busiest; -} - -/* - * pull_task - move a task from a remote runqueue to the local runqueue. - * Both runqueues must be locked. - */ -static inline void pull_task(runqueue_t *src_rq, prio_array_t *src_array, task_t *p, runqueue_t *this_rq, int this_cpu) -{ - dequeue_task(p, src_array); - nr_running_dec(src_rq); - set_task_cpu(p, this_cpu); - nr_running_inc(this_rq); - enqueue_task(p, this_rq->active); - /* - * Note that idle threads have a prio of MAX_PRIO, for this test - * to be always true for them. - */ - if (TASK_PREEMPTS_CURR(p, this_rq)) - set_need_resched(); -} - -/* - * Previously: - * - * #define CAN_MIGRATE_TASK(p,rq,this_cpu) \ - * ((!idle || (NS_TO_JIFFIES(now - (p)->timestamp) > \ - * cache_decay_ticks)) && !task_running(rq, p) && \ - * cpu_isset(this_cpu, (p)->cpus_allowed)) - */ - -static inline int -can_migrate_task(task_t *tsk, runqueue_t *rq, int this_cpu, int idle) -{ - unsigned long delta = sched_clock() - tsk->timestamp; - - if (!idle && (delta <= JIFFIES_TO_NS(cache_decay_ticks))) - return 0; - if (task_running(rq, tsk)) - return 0; - if (!cpu_isset(this_cpu, tsk->cpus_allowed)) - return 0; - return 1; -} - -/* - * Current runqueue is empty, or rebalance tick: if there is an - * inbalance (current runqueue is too short) then pull from - * busiest runqueue(s). - * - * We call this with the current runqueue locked, - * irqs disabled. - */ -static void load_balance(runqueue_t *this_rq, int idle, cpumask_t cpumask) -{ - int imbalance, idx, this_cpu = smp_processor_id(); - runqueue_t *busiest; - prio_array_t *array; - struct list_head *head, *curr; - task_t *tmp; - - busiest = find_busiest_queue(this_rq, this_cpu, idle, &imbalance, cpumask); - if (!busiest) - goto out; - - /* - * We only want to steal a number of tasks equal to 1/2 the imbalance, - * otherwise we'll just shift the imbalance to the new queue: - */ - imbalance /= 2; - - /* - * We first consider expired tasks. Those will likely not be - * executed in the near future, and they are most likely to - * be cache-cold, thus switching CPUs has the least effect - * on them. - */ - if (busiest->expired->nr_active) - array = busiest->expired; - else - array = busiest->active; - -new_array: - /* Start searching at priority 0: */ - idx = 0; -skip_bitmap: - if (!idx) - idx = sched_find_first_bit(array->bitmap); - else - idx = find_next_bit(array->bitmap, MAX_PRIO, idx); - if (idx >= MAX_PRIO) { - if (array == busiest->expired) { - array = busiest->active; - goto new_array; - } - goto out_unlock; - } - - head = array->queue + idx; - curr = head->prev; -skip_queue: - tmp = list_entry(curr, task_t, run_list); - - /* - * We do not migrate tasks that are: - * 1) running (obviously), or - * 2) cannot be migrated to this CPU due to cpus_allowed, or - * 3) are cache-hot on their current CPU. - */ - - curr = curr->prev; - - if (!can_migrate_task(tmp, busiest, this_cpu, idle)) { - if (curr != head) - goto skip_queue; - idx++; - goto skip_bitmap; - } - pull_task(busiest, array, tmp, this_rq, this_cpu); - if (!idle && --imbalance) { - if (curr != head) - goto skip_queue; - idx++; - goto skip_bitmap; - } -out_unlock: - spin_unlock(&busiest->lock); -out: - ; -} - -/* - * One of the idle_cpu_tick() and busy_cpu_tick() functions will - * get called every timer tick, on every CPU. Our balancing action - * frequency and balancing agressivity depends on whether the CPU is - * idle or not. - * - * busy-rebalance every 200 msecs. idle-rebalance every 1 msec. (or on - * systems with HZ=100, every 10 msecs.) - * - * On NUMA, do a node-rebalance every 400 msecs. - */ -#define IDLE_REBALANCE_TICK (HZ/1000 ?: 1) -#define BUSY_REBALANCE_TICK (HZ/5 ?: 1) -#define IDLE_NODE_REBALANCE_TICK (IDLE_REBALANCE_TICK * 5) -#define BUSY_NODE_REBALANCE_TICK (BUSY_REBALANCE_TICK * 2) - -#ifdef CONFIG_NUMA -static void balance_node(runqueue_t *this_rq, int idle, int this_cpu) -{ - int node = find_busiest_node(cpu_to_node(this_cpu)); - - if (node >= 0) { - cpumask_t cpumask = node_to_cpumask(node); - cpu_set(this_cpu, cpumask); - spin_lock(&this_rq->lock); - load_balance(this_rq, idle, cpumask); - spin_unlock(&this_rq->lock); - } -} -#endif - -static void rebalance_tick(runqueue_t *this_rq, int idle) -{ -#ifdef CONFIG_NUMA - int this_cpu = smp_processor_id(); -#endif - unsigned long j = jiffies; - - /* - * First do inter-node rebalancing, then intra-node rebalancing, - * if both events happen in the same tick. The inter-node - * rebalancing does not necessarily have to create a perfect - * balance within the node, since we load-balance the most loaded - * node with the current CPU. (ie. other CPUs in the local node - * are not balanced.) - */ - if (idle) { -#ifdef CONFIG_NUMA - if (!(j % IDLE_NODE_REBALANCE_TICK)) - balance_node(this_rq, idle, this_cpu); -#endif - if (!(j % IDLE_REBALANCE_TICK)) { - spin_lock(&this_rq->lock); - load_balance(this_rq, idle, cpu_to_node_mask(this_cpu)); - spin_unlock(&this_rq->lock); - } - return; - } -#ifdef CONFIG_NUMA - if (!(j % BUSY_NODE_REBALANCE_TICK)) - balance_node(this_rq, idle, this_cpu); -#endif - if (!(j % BUSY_REBALANCE_TICK)) { - spin_lock(&this_rq->lock); - load_balance(this_rq, idle, cpu_to_node_mask(this_cpu)); - spin_unlock(&this_rq->lock); - } -} -#else -/* - * on UP we do not need to balance between CPUs: - */ -static inline void rebalance_tick(runqueue_t *this_rq, int idle) -{ -} -#endif - -DEFINE_PER_CPU(struct kernel_stat, kstat) = { { 0 } }; - -EXPORT_PER_CPU_SYMBOL(kstat); - -/* - * We place interactive tasks back into the active array, if possible. - * - * To guarantee that this does not starve expired tasks we ignore the - * interactivity of a task if the first expired task had to wait more - * than a 'reasonable' amount of time. This deadline timeout is - * load-dependent, as the frequency of array switched decreases with - * increasing number of running tasks: - */ -#define EXPIRED_STARVING(rq) \ - (STARVATION_LIMIT && ((rq)->expired_timestamp && \ - (jiffies - (rq)->expired_timestamp >= \ - STARVATION_LIMIT * ((rq)->nr_running) + 1))) - -/* - * This function gets called by the timer code, with HZ frequency. - * We call it with interrupts disabled. - * - * It also gets called by the fork code, when changing the parent's - * timeslices. - */ -void scheduler_tick(int user_ticks, int sys_ticks) -{ - int cpu = smp_processor_id(); - struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat; - runqueue_t *rq = this_rq(); - task_t *p = current; - - if (rcu_pending(cpu)) - rcu_check_callbacks(cpu, user_ticks); - - /* note: this timer irq context must be accounted for as well */ - if (hardirq_count() - HARDIRQ_OFFSET) { - cpustat->irq += sys_ticks; - sys_ticks = 0; - } else if (softirq_count()) { - cpustat->softirq += sys_ticks; - sys_ticks = 0; - } - - if (p == rq->idle) { - if (atomic_read(&rq->nr_iowait) > 0) - cpustat->iowait += sys_ticks; - else - cpustat->idle += sys_ticks; - rebalance_tick(rq, 1); - return; - } - if (TASK_NICE(p) > 0) - cpustat->nice += user_ticks; - else - cpustat->user += user_ticks; - cpustat->system += sys_ticks; - - /* Task might have expired already, but not scheduled off yet */ - if (p->array != rq->active) { - set_tsk_need_resched(p); - goto out; - } - spin_lock(&rq->lock); - /* - * The task was running during this tick - update the - * time slice counter. Note: we do not update a thread's - * priority until it either goes to sleep or uses up its - * timeslice. This makes it possible for interactive tasks - * to use up their timeslices at their highest priority levels. - */ - if (unlikely(rt_task(p))) { - /* - * RR tasks need a special form of timeslice management. - * FIFO tasks have no timeslices. - */ - if ((p->policy == SCHED_RR) && !--p->time_slice) { - p->time_slice = task_timeslice(p); - p->first_time_slice = 0; - set_tsk_need_resched(p); - - /* put it at the end of the queue: */ - dequeue_task(p, rq->active); - enqueue_task(p, rq->active); - } - goto out_unlock; - } - if (!--p->time_slice) { - dequeue_task(p, rq->active); - set_tsk_need_resched(p); - p->prio = effective_prio(p); - p->time_slice = task_timeslice(p); - p->first_time_slice = 0; - - if (!rq->expired_timestamp) - rq->expired_timestamp = jiffies; - if (!TASK_INTERACTIVE(p) || EXPIRED_STARVING(rq)) { - enqueue_task(p, rq->expired); - } else - enqueue_task(p, rq->active); - } else { - /* - * Prevent a too long timeslice allowing a task to monopolize - * the CPU. We do this by splitting up the timeslice into - * smaller pieces. - * - * Note: this does not mean the task's timeslices expire or - * get lost in any way, they just might be preempted by - * another task of equal priority. (one with higher - * priority would have preempted this task already.) We - * requeue this task to the end of the list on this priority - * level, which is in essence a round-robin of tasks with - * equal priority. - * - * This only applies to tasks in the interactive - * delta range with at least TIMESLICE_GRANULARITY to requeue. - */ - if (TASK_INTERACTIVE(p) && !((task_timeslice(p) - - p->time_slice) % TIMESLICE_GRANULARITY(p)) && - (p->time_slice >= TIMESLICE_GRANULARITY(p)) && - (p->array == rq->active)) { - - dequeue_task(p, rq->active); - set_tsk_need_resched(p); - p->prio = effective_prio(p); - enqueue_task(p, rq->active); - } - } -out_unlock: - spin_unlock(&rq->lock); -out: - rebalance_tick(rq, 0); -} - -void scheduling_functions_start_here(void) { } - -/* - * schedule() is the main scheduler function. - */ -asmlinkage void schedule(void) -{ - task_t *prev, *next; - runqueue_t *rq; - prio_array_t *array; - struct list_head *queue; - unsigned long long now; - unsigned long run_time; - int idx; - - /* - * Test if we are atomic. Since do_exit() needs to call into - * schedule() atomically, we ignore that path for now. - * Otherwise, whine if we are scheduling when we should not be. - */ - if (likely(!(current->state & (TASK_DEAD | TASK_ZOMBIE)))) { - if (unlikely(in_atomic())) { - printk(KERN_ERR "bad: scheduling while atomic!\n"); - dump_stack(); - } - } - -need_resched: - preempt_disable(); - prev = current; - rq = this_rq(); - - release_kernel_lock(prev); - now = sched_clock(); - if (likely(now - prev->timestamp < NS_MAX_SLEEP_AVG)) - run_time = now - prev->timestamp; - else - run_time = NS_MAX_SLEEP_AVG; - - /* - * Tasks with interactive credits get charged less run_time - * at high sleep_avg to delay them losing their interactive - * status - */ - if (HIGH_CREDIT(prev)) - run_time /= (CURRENT_BONUS(prev) ? : 1); - - spin_lock_irq(&rq->lock); - - /* - * if entering off of a kernel preemption go straight - * to picking the next task. - */ - if (unlikely(preempt_count() & PREEMPT_ACTIVE)) { - prev->nivcsw++; - goto pick_next_task; - } - - switch (prev->state) { - case TASK_INTERRUPTIBLE: - if (unlikely(signal_pending(prev))) { - prev->state = TASK_RUNNING; - break; - } - default: - deactivate_task(prev, rq); - prev->nvcsw++; - break; - case TASK_RUNNING: - prev->nivcsw++; - } -pick_next_task: - if (unlikely(!rq->nr_running)) { -#ifdef CONFIG_SMP - load_balance(rq, 1, cpu_to_node_mask(smp_processor_id())); - if (rq->nr_running) - goto pick_next_task; -#endif - next = rq->idle; - rq->expired_timestamp = 0; - goto switch_tasks; - } - - array = rq->active; - if (unlikely(!array->nr_active)) { - /* - * Switch the active and expired arrays. - */ - rq->active = rq->expired; - rq->expired = array; - array = rq->active; - rq->expired_timestamp = 0; - } - - idx = sched_find_first_bit(array->bitmap); - queue = array->queue + idx; - next = list_entry(queue->next, task_t, run_list); - - if (next->activated > 0) { - unsigned long long delta = now - next->timestamp; - - if (next->activated == 1) - delta = delta * (ON_RUNQUEUE_WEIGHT * 128 / 100) / 128; - - array = next->array; - dequeue_task(next, array); - recalc_task_prio(next, next->timestamp + delta); - enqueue_task(next, array); - } - next->activated = 0; -switch_tasks: - prefetch(next); - clear_tsk_need_resched(prev); - RCU_qsctr(task_cpu(prev))++; - - prev->sleep_avg -= run_time; - if ((long)prev->sleep_avg <= 0){ - prev->sleep_avg = 0; - if (!(HIGH_CREDIT(prev) || LOW_CREDIT(prev))) - prev->interactive_credit--; - } - prev->timestamp = now; - - if (likely(prev != next)) { - next->timestamp = now; - rq->nr_switches++; - rq->curr = next; - - prepare_arch_switch(rq, next); - prev = context_switch(rq, prev, next); - barrier(); - - finish_task_switch(prev); - } else { - /* - * A switch to the same task doesn't count as - * a context switch - */ - prev->nivcsw--; - spin_unlock_irq(&rq->lock); - } - - reacquire_kernel_lock(current); - preempt_enable_no_resched(); - if (test_thread_flag(TIF_NEED_RESCHED)) - goto need_resched; -} - -EXPORT_SYMBOL(schedule); - -#ifdef CONFIG_PREEMPT -/* - * this is is the entry point to schedule() from in-kernel preemption - * off of preempt_enable. Kernel preemptions off return from interrupt - * occur there and call schedule directly. - */ -asmlinkage void preempt_schedule(void) -{ - struct thread_info *ti = current_thread_info(); - - /* - * If there is a non-zero preempt_count or interrupts are disabled, - * we do not want to preempt the current task. Just return.. - */ - if (unlikely(ti->preempt_count || irqs_disabled())) - return; - -need_resched: - ti->preempt_count = PREEMPT_ACTIVE; - schedule(); - ti->preempt_count = 0; - - /* we could miss a preemption opportunity between schedule and now */ - barrier(); - if (unlikely(test_thread_flag(TIF_NEED_RESCHED))) - goto need_resched; -} - -EXPORT_SYMBOL(preempt_schedule); -#endif /* CONFIG_PREEMPT */ - -int default_wake_function(wait_queue_t *curr, unsigned mode, int sync) -{ - task_t *p = curr->task; - return try_to_wake_up(p, mode, sync, 0); -} - -EXPORT_SYMBOL(default_wake_function); - -/* - * The core wakeup function. Non-exclusive wakeups (nr_exclusive == 0) just - * wake everything up. If it's an exclusive wakeup (nr_exclusive == small +ve - * number) then we wake all the non-exclusive tasks and one exclusive task. - * - * There are circumstances in which we can try to wake a task which has already - * started to run but is not in state TASK_RUNNING. try_to_wake_up() returns - * zero in this (rare) case, and we handle it by continuing to scan the queue. - */ -static void __wake_up_common(wait_queue_head_t *q, unsigned int mode, int nr_exclusive, int sync) -{ - struct list_head *tmp, *next; - - list_for_each_safe(tmp, next, &q->task_list) { - wait_queue_t *curr; - unsigned flags; - curr = list_entry(tmp, wait_queue_t, task_list); - flags = curr->flags; - if (curr->func(curr, mode, sync) && - (flags & WQ_FLAG_EXCLUSIVE) && - !--nr_exclusive) - break; - } -} - -/** - * __wake_up - wake up threads blocked on a waitqueue. - * @q: the waitqueue - * @mode: which threads - * @nr_exclusive: how many wake-one or wake-many threads to wake up - */ -void __wake_up(wait_queue_head_t *q, unsigned int mode, int nr_exclusive) -{ - unsigned long flags; - - spin_lock_irqsave(&q->lock, flags); - __wake_up_common(q, mode, nr_exclusive, 0); - spin_unlock_irqrestore(&q->lock, flags); -} - -EXPORT_SYMBOL(__wake_up); - -/* - * Same as __wake_up but called with the spinlock in wait_queue_head_t held. - */ -void __wake_up_locked(wait_queue_head_t *q, unsigned int mode) -{ - __wake_up_common(q, mode, 1, 0); -} - -/** - * __wake_up - sync- wake up threads blocked on a waitqueue. - * @q: the waitqueue - * @mode: which threads - * @nr_exclusive: how many wake-one or wake-many threads to wake up - * - * The sync wakeup differs that the waker knows that it will schedule - * away soon, so while the target thread will be woken up, it will not - * be migrated to another CPU - ie. the two threads are 'synchronized' - * with each other. This can prevent needless bouncing between CPUs. - * - * On UP it can prevent extra preemption. - */ -void __wake_up_sync(wait_queue_head_t *q, unsigned int mode, int nr_exclusive) -{ - unsigned long flags; - - if (unlikely(!q)) - return; - - spin_lock_irqsave(&q->lock, flags); - if (likely(nr_exclusive)) - __wake_up_common(q, mode, nr_exclusive, 1); - else - __wake_up_common(q, mode, nr_exclusive, 0); - spin_unlock_irqrestore(&q->lock, flags); -} - -EXPORT_SYMBOL_GPL(__wake_up_sync); /* For internal use only */ - -void complete(struct completion *x) -{ - unsigned long flags; - - spin_lock_irqsave(&x->wait.lock, flags); - x->done++; - __wake_up_common(&x->wait, TASK_UNINTERRUPTIBLE | TASK_INTERRUPTIBLE, 1, 0); - spin_unlock_irqrestore(&x->wait.lock, flags); -} - -EXPORT_SYMBOL(complete); - -void complete_all(struct completion *x) -{ - unsigned long flags; - - spin_lock_irqsave(&x->wait.lock, flags); - x->done += UINT_MAX/2; - __wake_up_common(&x->wait, TASK_UNINTERRUPTIBLE | TASK_INTERRUPTIBLE, 0, 0); - spin_unlock_irqrestore(&x->wait.lock, flags); -} - -void wait_for_completion(struct completion *x) -{ - might_sleep(); - spin_lock_irq(&x->wait.lock); - if (!x->done) { - DECLARE_WAITQUEUE(wait, current); - - wait.flags |= WQ_FLAG_EXCLUSIVE; - __add_wait_queue_tail(&x->wait, &wait); - do { - __set_current_state(TASK_UNINTERRUPTIBLE); - spin_unlock_irq(&x->wait.lock); - schedule(); - spin_lock_irq(&x->wait.lock); - } while (!x->done); - __remove_wait_queue(&x->wait, &wait); - } - x->done--; - spin_unlock_irq(&x->wait.lock); -} - -EXPORT_SYMBOL(wait_for_completion); - -#define SLEEP_ON_VAR \ - unsigned long flags; \ - wait_queue_t wait; \ - init_waitqueue_entry(&wait, current); - -#define SLEEP_ON_HEAD \ - spin_lock_irqsave(&q->lock,flags); \ - __add_wait_queue(q, &wait); \ - spin_unlock(&q->lock); - -#define SLEEP_ON_TAIL \ - spin_lock_irq(&q->lock); \ - __remove_wait_queue(q, &wait); \ - spin_unlock_irqrestore(&q->lock, flags); - -void interruptible_sleep_on(wait_queue_head_t *q) -{ - SLEEP_ON_VAR - - current->state = TASK_INTERRUPTIBLE; - - SLEEP_ON_HEAD - schedule(); - SLEEP_ON_TAIL -} - -EXPORT_SYMBOL(interruptible_sleep_on); - -long interruptible_sleep_on_timeout(wait_queue_head_t *q, long timeout) -{ - SLEEP_ON_VAR - - current->state = TASK_INTERRUPTIBLE; - - SLEEP_ON_HEAD - timeout = schedule_timeout(timeout); - SLEEP_ON_TAIL - - return timeout; -} - -EXPORT_SYMBOL(interruptible_sleep_on_timeout); - -void sleep_on(wait_queue_head_t *q) -{ - SLEEP_ON_VAR - - current->state = TASK_UNINTERRUPTIBLE; - - SLEEP_ON_HEAD - schedule(); - SLEEP_ON_TAIL -} - -EXPORT_SYMBOL(sleep_on); - -long sleep_on_timeout(wait_queue_head_t *q, long timeout) -{ - SLEEP_ON_VAR - - current->state = TASK_UNINTERRUPTIBLE; - - SLEEP_ON_HEAD - timeout = schedule_timeout(timeout); - SLEEP_ON_TAIL - - return timeout; -} - -EXPORT_SYMBOL(sleep_on_timeout); - -void scheduling_functions_end_here(void) { } - -void set_user_nice(task_t *p, long nice) -{ - unsigned long flags; - prio_array_t *array; - runqueue_t *rq; - int old_prio, new_prio, delta; - - if (TASK_NICE(p) == nice || nice < -20 || nice > 19) - return; - /* - * We have to be careful, if called from sys_setpriority(), - * the task might be in the middle of scheduling on another CPU. - */ - rq = task_rq_lock(p, &flags); - /* - * The RT priorities are set via setscheduler(), but we still - * allow the 'normal' nice value to be set - but as expected - * it wont have any effect on scheduling until the task is - * not SCHED_NORMAL: - */ - if (rt_task(p)) { - p->static_prio = NICE_TO_PRIO(nice); - goto out_unlock; - } - array = p->array; - if (array) - dequeue_task(p, array); - - old_prio = p->prio; - new_prio = NICE_TO_PRIO(nice); - delta = new_prio - old_prio; - p->static_prio = NICE_TO_PRIO(nice); - p->prio += delta; - - if (array) { - enqueue_task(p, array); - /* - * If the task increased its priority or is running and - * lowered its priority, then reschedule its CPU: - */ - if (delta < 0 || (delta > 0 && task_running(rq, p))) - resched_task(rq->curr); - } -out_unlock: - task_rq_unlock(rq, &flags); -} - -EXPORT_SYMBOL(set_user_nice); - -#ifndef __alpha__ - -/* - * sys_nice - change the priority of the current process. - * @increment: priority increment - * - * sys_setpriority is a more generic, but much slower function that - * does similar things. - */ -asmlinkage long sys_nice(int increment) -{ - int retval; - long nice; - - /* - * Setpriority might change our priority at the same moment. - * We don't have to worry. Conceptually one call occurs first - * and we have a single winner. - */ - if (increment < 0) { - if (!capable(CAP_SYS_NICE)) - return -EPERM; - if (increment < -40) - increment = -40; - } - if (increment > 40) - increment = 40; - - nice = PRIO_TO_NICE(current->static_prio) + increment; - if (nice < -20) - nice = -20; - if (nice > 19) - nice = 19; - - retval = security_task_setnice(current, nice); - if (retval) - return retval; - - set_user_nice(current, nice); - return 0; -} - -#endif - -/** - * task_prio - return the priority value of a given task. - * @p: the task in question. - * - * This is the priority value as seen by users in /proc. - * RT tasks are offset by -200. Normal tasks are centered - * around 0, value goes from -16 to +15. - */ -int task_prio(task_t *p) -{ - return p->prio - MAX_RT_PRIO; -} - -/** - * task_nice - return the nice value of a given task. - * @p: the task in question. - */ -int task_nice(task_t *p) -{ - return TASK_NICE(p); -} - -EXPORT_SYMBOL(task_nice); - -/** - * task_curr - is this task currently executing on a CPU? - * @p: the task in question. - */ -int task_curr(task_t *p) -{ - return cpu_curr(task_cpu(p)) == p; -} - -/** - * idle_cpu - is a given cpu idle currently? - * @cpu: the processor in question. - */ -int idle_cpu(int cpu) -{ - return cpu_curr(cpu) == cpu_rq(cpu)->idle; -} - -EXPORT_SYMBOL_GPL(idle_cpu); - -/** - * find_process_by_pid - find a process with a matching PID value. - * @pid: the pid in question. - */ -static inline task_t *find_process_by_pid(pid_t pid) -{ - return pid ? find_task_by_pid(pid) : current; -} - -/* - * setscheduler - change the scheduling policy and/or RT priority of a thread. - */ -static int setscheduler(pid_t pid, int policy, struct sched_param __user *param) -{ - struct sched_param lp; - int retval = -EINVAL; - int oldprio; - prio_array_t *array; - unsigned long flags; - runqueue_t *rq; - task_t *p; - - if (!param || pid < 0) - goto out_nounlock; - - retval = -EFAULT; - if (copy_from_user(&lp, param, sizeof(struct sched_param))) - goto out_nounlock; - - /* - * We play safe to avoid deadlocks. - */ - read_lock_irq(&tasklist_lock); - - p = find_process_by_pid(pid); - - retval = -ESRCH; - if (!p) - goto out_unlock_tasklist; - - /* - * To be able to change p->policy safely, the apropriate - * runqueue lock must be held. - */ - rq = task_rq_lock(p, &flags); - - if (policy < 0) - policy = p->policy; - else { - retval = -EINVAL; - if (policy != SCHED_FIFO && policy != SCHED_RR && - policy != SCHED_NORMAL) - goto out_unlock; - } - - /* - * Valid priorities for SCHED_FIFO and SCHED_RR are - * 1..MAX_USER_RT_PRIO-1, valid priority for SCHED_NORMAL is 0. - */ - retval = -EINVAL; - if (lp.sched_priority < 0 || lp.sched_priority > MAX_USER_RT_PRIO-1) - goto out_unlock; - if ((policy == SCHED_NORMAL) != (lp.sched_priority == 0)) - goto out_unlock; - - retval = -EPERM; - if ((policy == SCHED_FIFO || policy == SCHED_RR) && - !capable(CAP_SYS_NICE)) - goto out_unlock; - if ((current->euid != p->euid) && (current->euid != p->uid) && - !capable(CAP_SYS_NICE)) - goto out_unlock; - - retval = security_task_setscheduler(p, policy, &lp); - if (retval) - goto out_unlock; - - array = p->array; - if (array) - deactivate_task(p, task_rq(p)); - retval = 0; - p->policy = policy; - p->rt_priority = lp.sched_priority; - oldprio = p->prio; - if (policy != SCHED_NORMAL) - p->prio = MAX_USER_RT_PRIO-1 - p->rt_priority; - else - p->prio = p->static_prio; - if (array) { - __activate_task(p, task_rq(p)); - /* - * Reschedule if we are currently running on this runqueue and - * our priority decreased, or if we are not currently running on - * this runqueue and our priority is higher than the current's - */ - if (rq->curr == p) { - if (p->prio > oldprio) - resched_task(rq->curr); - } else if (p->prio < rq->curr->prio) - resched_task(rq->curr); - } - -out_unlock: - task_rq_unlock(rq, &flags); -out_unlock_tasklist: - read_unlock_irq(&tasklist_lock); - -out_nounlock: - return retval; -} - -/** - * sys_sched_setscheduler - set/change the scheduler policy and RT priority - * @pid: the pid in question. - * @policy: new policy - * @param: structure containing the new RT priority. - */ -asmlinkage long sys_sched_setscheduler(pid_t pid, int policy, - struct sched_param __user *param) -{ - return setscheduler(pid, policy, param); -} - -/** - * sys_sched_setparam - set/change the RT priority of a thread - * @pid: the pid in question. - * @param: structure containing the new RT priority. - */ -asmlinkage long sys_sched_setparam(pid_t pid, struct sched_param __user *param) -{ - return setscheduler(pid, -1, param); -} - -/** - * sys_sched_getscheduler - get the policy (scheduling class) of a thread - * @pid: the pid in question. - */ -asmlinkage long sys_sched_getscheduler(pid_t pid) -{ - int retval = -EINVAL; - task_t *p; - - if (pid < 0) - goto out_nounlock; - - retval = -ESRCH; - read_lock(&tasklist_lock); - p = find_process_by_pid(pid); - if (p) { - retval = security_task_getscheduler(p); - if (!retval) - retval = p->policy; - } - read_unlock(&tasklist_lock); - -out_nounlock: - return retval; -} - -/** - * sys_sched_getscheduler - get the RT priority of a thread - * @pid: the pid in question. - * @param: structure containing the RT priority. - */ -asmlinkage long sys_sched_getparam(pid_t pid, struct sched_param __user *param) -{ - struct sched_param lp; - int retval = -EINVAL; - task_t *p; - - if (!param || pid < 0) - goto out_nounlock; - - read_lock(&tasklist_lock); - p = find_process_by_pid(pid); - retval = -ESRCH; - if (!p) - goto out_unlock; - - retval = security_task_getscheduler(p); - if (retval) - goto out_unlock; - - lp.sched_priority = p->rt_priority; - read_unlock(&tasklist_lock); - - /* - * This one might sleep, we cannot do it with a spinlock held ... - */ - retval = copy_to_user(param, &lp, sizeof(*param)) ? -EFAULT : 0; - -out_nounlock: - return retval; - -out_unlock: - read_unlock(&tasklist_lock); - return retval; -} - -/** - * sys_sched_setaffinity - set the cpu affinity of a process - * @pid: pid of the process - * @len: length in bytes of the bitmask pointed to by user_mask_ptr - * @user_mask_ptr: user-space pointer to the new cpu mask - */ -asmlinkage long sys_sched_setaffinity(pid_t pid, unsigned int len, - unsigned long __user *user_mask_ptr) -{ - cpumask_t new_mask; - int retval; - task_t *p; - - if (len < sizeof(new_mask)) - return -EINVAL; - - if (copy_from_user(&new_mask, user_mask_ptr, sizeof(new_mask))) - return -EFAULT; - - read_lock(&tasklist_lock); - - p = find_process_by_pid(pid); - if (!p) { - read_unlock(&tasklist_lock); - return -ESRCH; - } - - /* - * It is not safe to call set_cpus_allowed with the - * tasklist_lock held. We will bump the task_struct's - * usage count and then drop tasklist_lock. - */ - get_task_struct(p); - read_unlock(&tasklist_lock); - - retval = -EPERM; - if ((current->euid != p->euid) && (current->euid != p->uid) && - !capable(CAP_SYS_NICE)) - goto out_unlock; - - retval = set_cpus_allowed(p, new_mask); - -out_unlock: - put_task_struct(p); - return retval; -} - -/** - * sys_sched_getaffinity - get the cpu affinity of a process - * @pid: pid of the process - * @len: length in bytes of the bitmask pointed to by user_mask_ptr - * @user_mask_ptr: user-space pointer to hold the current cpu mask - */ -asmlinkage long sys_sched_getaffinity(pid_t pid, unsigned int len, - unsigned long __user *user_mask_ptr) -{ - unsigned int real_len; - cpumask_t mask; - int retval; - task_t *p; - - real_len = sizeof(mask); - if (len < real_len) - return -EINVAL; - - read_lock(&tasklist_lock); - - retval = -ESRCH; - p = find_process_by_pid(pid); - if (!p) - goto out_unlock; - - retval = 0; - cpus_and(mask, p->cpus_allowed, cpu_online_map); - -out_unlock: - read_unlock(&tasklist_lock); - if (retval) - return retval; - if (copy_to_user(user_mask_ptr, &mask, real_len)) - return -EFAULT; - return real_len; -} - -/** - * sys_sched_yield - yield the current processor to other threads. - * - * this function yields the current CPU by moving the calling thread - * to the expired array. If there are no other threads running on this - * CPU then this function will return. - */ -asmlinkage long sys_sched_yield(void) -{ - runqueue_t *rq = this_rq_lock(); - prio_array_t *array = current->array; - - /* - * We implement yielding by moving the task into the expired - * queue. - * - * (special rule: RT tasks will just roundrobin in the active - * array.) - */ - if (likely(!rt_task(current))) { - dequeue_task(current, array); - enqueue_task(current, rq->expired); - } else { - list_del(¤t->run_list); - list_add_tail(¤t->run_list, array->queue + current->prio); - } - /* - * Since we are going to call schedule() anyway, there's - * no need to preempt: - */ - _raw_spin_unlock(&rq->lock); - preempt_enable_no_resched(); - - schedule(); - - return 0; -} - -void __cond_resched(void) -{ - set_current_state(TASK_RUNNING); - schedule(); -} - -EXPORT_SYMBOL(__cond_resched); - -/** - * yield - yield the current processor to other threads. - * - * this is a shortcut for kernel-space yielding - it marks the - * thread runnable and calls sys_sched_yield(). - */ -void yield(void) -{ - set_current_state(TASK_RUNNING); - sys_sched_yield(); -} - -EXPORT_SYMBOL(yield); - -/* - * This task is about to go to sleep on IO. Increment rq->nr_iowait so - * that process accounting knows that this is a task in IO wait state. - * - * But don't do that if it is a deliberate, throttling IO wait (this task - * has set its backing_dev_info: the queue against which it should throttle) - */ -void io_schedule(void) -{ - struct runqueue *rq = this_rq(); - - atomic_inc(&rq->nr_iowait); - schedule(); - atomic_dec(&rq->nr_iowait); -} - -EXPORT_SYMBOL(io_schedule); - -long io_schedule_timeout(long timeout) -{ - struct runqueue *rq = this_rq(); - long ret; - - atomic_inc(&rq->nr_iowait); - ret = schedule_timeout(timeout); - atomic_dec(&rq->nr_iowait); - return ret; -} - -/** - * sys_sched_get_priority_max - return maximum RT priority. - * @policy: scheduling class. - * - * this syscall returns the maximum rt_priority that can be used - * by a given scheduling class. - */ -asmlinkage long sys_sched_get_priority_max(int policy) -{ - int ret = -EINVAL; - - switch (policy) { - case SCHED_FIFO: - case SCHED_RR: - ret = MAX_USER_RT_PRIO-1; - break; - case SCHED_NORMAL: - ret = 0; - break; - } - return ret; -} - -/** - * sys_sched_get_priority_min - return minimum RT priority. - * @policy: scheduling class. - * - * this syscall returns the minimum rt_priority that can be used - * by a given scheduling class. - */ -asmlinkage long sys_sched_get_priority_min(int policy) -{ - int ret = -EINVAL; - - switch (policy) { - case SCHED_FIFO: - case SCHED_RR: - ret = 1; - break; - case SCHED_NORMAL: - ret = 0; - } - return ret; -} - -/** - * sys_sched_rr_get_interval - return the default timeslice of a process. - * @pid: pid of the process. - * @interval: userspace pointer to the timeslice value. - * - * this syscall writes the default timeslice value of a given process - * into the user-space timespec buffer. A value of '0' means infinity. - */ -asmlinkage long sys_sched_rr_get_interval(pid_t pid, struct timespec __user *interval) -{ - int retval = -EINVAL; - struct timespec t; - task_t *p; - - if (pid < 0) - goto out_nounlock; - - retval = -ESRCH; - read_lock(&tasklist_lock); - p = find_process_by_pid(pid); - if (!p) - goto out_unlock; - - retval = security_task_getscheduler(p); - if (retval) - goto out_unlock; - - jiffies_to_timespec(p->policy & SCHED_FIFO ? - 0 : task_timeslice(p), &t); - read_unlock(&tasklist_lock); - retval = copy_to_user(interval, &t, sizeof(t)) ? -EFAULT : 0; -out_nounlock: - return retval; -out_unlock: - read_unlock(&tasklist_lock); - return retval; -} - -static inline struct task_struct *eldest_child(struct task_struct *p) -{ - if (list_empty(&p->children)) return NULL; - return list_entry(p->children.next,struct task_struct,sibling); -} - -static inline struct task_struct *older_sibling(struct task_struct *p) -{ - if (p->sibling.prev==&p->parent->children) return NULL; - return list_entry(p->sibling.prev,struct task_struct,sibling); -} - -static inline struct task_struct *younger_sibling(struct task_struct *p) -{ - if (p->sibling.next==&p->parent->children) return NULL; - return list_entry(p->sibling.next,struct task_struct,sibling); -} - -static void show_task(task_t * p) -{ - unsigned long free = 0; - task_t *relative; - int state; - static const char * stat_nam[] = { "R", "S", "D", "T", "Z", "W" }; - - printk("%-13.13s ", p->comm); - state = p->state ? __ffs(p->state) + 1 : 0; - if (((unsigned) state) < sizeof(stat_nam)/sizeof(char *)) - printk(stat_nam[state]); - else - printk(" "); -#if (BITS_PER_LONG == 32) - if (p == current) - printk(" current "); - else - printk(" %08lX ", thread_saved_pc(p)); -#else - if (p == current) - printk(" current task "); - else - printk(" %016lx ", thread_saved_pc(p)); -#endif - { - unsigned long * n = (unsigned long *) (p->thread_info+1); - while (!*n) - n++; - free = (unsigned long) n - (unsigned long)(p->thread_info+1); - } - printk("%5lu %5d %6d ", free, p->pid, p->parent->pid); - if ((relative = eldest_child(p))) - printk("%5d ", relative->pid); - else - printk(" "); - if ((relative = younger_sibling(p))) - printk("%7d", relative->pid); - else - printk(" "); - if ((relative = older_sibling(p))) - printk(" %5d", relative->pid); - else - printk(" "); - if (!p->mm) - printk(" (L-TLB)\n"); - else - printk(" (NOTLB)\n"); - - show_stack(p, NULL); -} - -void show_state(void) -{ - task_t *g, *p; - -#if (BITS_PER_LONG == 32) - printk("\n" - " free sibling\n"); - printk(" task PC stack pid father child younger older\n"); -#else - printk("\n" - " free sibling\n"); - printk(" task PC stack pid father child younger older\n"); -#endif - read_lock(&tasklist_lock); - do_each_thread(g, p) { - /* - * reset the NMI-timeout, listing all files on a slow - * console might take alot of time: - */ - touch_nmi_watchdog(); - show_task(p); - } while_each_thread(g, p); - - read_unlock(&tasklist_lock); -} - -void __init init_idle(task_t *idle, int cpu) -{ - runqueue_t *idle_rq = cpu_rq(cpu), *rq = cpu_rq(task_cpu(idle)); - unsigned long flags; - - local_irq_save(flags); - double_rq_lock(idle_rq, rq); - - idle_rq->curr = idle_rq->idle = idle; - deactivate_task(idle, rq); - idle->array = NULL; - idle->prio = MAX_PRIO; - idle->state = TASK_RUNNING; - set_task_cpu(idle, cpu); - double_rq_unlock(idle_rq, rq); - set_tsk_need_resched(idle); - local_irq_restore(flags); - - /* Set the preempt count _outside_ the spinlocks! */ -#ifdef CONFIG_PREEMPT - idle->thread_info->preempt_count = (idle->lock_depth >= 0); -#else - idle->thread_info->preempt_count = 0; -#endif -} - -#ifdef CONFIG_SMP -/* - * This is how migration works: - * - * 1) we queue a migration_req_t structure in the source CPU's - * runqueue and wake up that CPU's migration thread. - * 2) we down() the locked semaphore => thread blocks. - * 3) migration thread wakes up (implicitly it forces the migrated - * thread off the CPU) - * 4) it gets the migration request and checks whether the migrated - * task is still in the wrong runqueue. - * 5) if it's in the wrong runqueue then the migration thread removes - * it and puts it into the right queue. - * 6) migration thread up()s the semaphore. - * 7) we wake up and the migration is done. - */ - -typedef struct { - struct list_head list; - task_t *task; - struct completion done; -} migration_req_t; - -/* - * Change a given task's CPU affinity. Migrate the thread to a - * proper CPU and schedule it away if the CPU it's executing on - * is removed from the allowed bitmask. - * - * NOTE: the caller must have a valid reference to the task, the - * task must not exit() & deallocate itself prematurely. The - * call is not atomic; no spinlocks may be held. - */ -int set_cpus_allowed(task_t *p, cpumask_t new_mask) -{ - unsigned long flags; - migration_req_t req; - runqueue_t *rq; - - if (any_online_cpu(new_mask) == NR_CPUS) - return -EINVAL; - - rq = task_rq_lock(p, &flags); - p->cpus_allowed = new_mask; - /* - * Can the task run on the task's current CPU? If not then - * migrate the thread off to a proper CPU. - */ - if (cpu_isset(task_cpu(p), new_mask)) { - task_rq_unlock(rq, &flags); - return 0; - } - /* - * If the task is not on a runqueue (and not running), then - * it is sufficient to simply update the task's cpu field. - */ - if (!p->array && !task_running(rq, p)) { - set_task_cpu(p, any_online_cpu(p->cpus_allowed)); - task_rq_unlock(rq, &flags); - return 0; - } - init_completion(&req.done); - req.task = p; - list_add(&req.list, &rq->migration_queue); - task_rq_unlock(rq, &flags); - - wake_up_process(rq->migration_thread); - - wait_for_completion(&req.done); - return 0; -} - -EXPORT_SYMBOL_GPL(set_cpus_allowed); - -/* Move (not current) task off this cpu, onto dest cpu. */ -static void move_task_away(struct task_struct *p, int dest_cpu) -{ - runqueue_t *rq_dest; - unsigned long flags; - - rq_dest = cpu_rq(dest_cpu); - - local_irq_save(flags); - double_rq_lock(this_rq(), rq_dest); - if (task_cpu(p) != smp_processor_id()) - goto out; /* Already moved */ - - set_task_cpu(p, dest_cpu); - if (p->array) { - deactivate_task(p, this_rq()); - activate_task(p, rq_dest); - if (p->prio < rq_dest->curr->prio) - resched_task(rq_dest->curr); - } - out: - double_rq_unlock(this_rq(), rq_dest); - local_irq_restore(flags); -} - -typedef struct { - int cpu; - struct completion startup_done; - task_t *task; -} migration_startup_t; - -/* - * migration_thread - this is a highprio system thread that performs - * thread migration by bumping thread off CPU then 'pushing' onto - * another runqueue. - */ -static int migration_thread(void * data) -{ - /* Marking "param" __user is ok, since we do a set_fs(KERNEL_DS); */ - struct sched_param __user param = { .sched_priority = MAX_RT_PRIO-1 }; - migration_startup_t *startup = data; - int cpu = startup->cpu; - runqueue_t *rq; - int ret; - - startup->task = current; - complete(&startup->startup_done); - set_current_state(TASK_UNINTERRUPTIBLE); - schedule(); - - BUG_ON(smp_processor_id() != cpu); - - daemonize("migration/%d", cpu); - set_fs(KERNEL_DS); - - ret = setscheduler(0, SCHED_FIFO, ¶m); - - rq = this_rq(); - rq->migration_thread = current; - - for (;;) { - struct list_head *head; - migration_req_t *req; - - if (current->flags & PF_FREEZE) - refrigerator(PF_IOTHREAD); - - spin_lock_irq(&rq->lock); - head = &rq->migration_queue; - current->state = TASK_INTERRUPTIBLE; - if (list_empty(head)) { - spin_unlock_irq(&rq->lock); - schedule(); - continue; - } - req = list_entry(head->next, migration_req_t, list); - list_del_init(head->next); - spin_unlock_irq(&rq->lock); - - move_task_away(req->task, - any_online_cpu(req->task->cpus_allowed)); - complete(&req->done); - } -} - -/* - * migration_call - callback that gets triggered when a CPU is added. - * Here we can start up the necessary migration thread for the new CPU. - */ -static int migration_call(struct notifier_block *nfb, - unsigned long action, - void *hcpu) -{ - long cpu = (long) hcpu; - migration_startup_t startup; - - switch (action) { - case CPU_ONLINE: - - printk("Starting migration thread for cpu %li\n", cpu); - - startup.cpu = cpu; - startup.task = NULL; - init_completion(&startup.startup_done); - - kernel_thread(migration_thread, &startup, CLONE_KERNEL); - wait_for_completion(&startup.startup_done); - wait_task_inactive(startup.task); - - startup.task->thread_info->cpu = cpu; - startup.task->cpus_allowed = cpumask_of_cpu(cpu); - - wake_up_process(startup.task); - - while (!cpu_rq(cpu)->migration_thread) - yield(); - - break; - } - return NOTIFY_OK; -} - -static struct notifier_block migration_notifier = { &migration_call, NULL, 0 }; - -__init int migration_init(void) -{ - /* Start one for boot CPU. */ - migration_call(&migration_notifier, CPU_ONLINE, - (void *)(long)smp_processor_id()); - register_cpu_notifier(&migration_notifier); - return 0; -} - -#endif - -#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT) -/* - * The 'big kernel lock' - * - * This spinlock is taken and released recursively by lock_kernel() - * and unlock_kernel(). It is transparently dropped and reaquired - * over schedule(). It is used to protect legacy code that hasn't - * been migrated to a proper locking design yet. - * - * Don't use in new code. - */ -spinlock_t kernel_flag __cacheline_aligned_in_smp = SPIN_LOCK_UNLOCKED; - -EXPORT_SYMBOL(kernel_flag); -#endif - -static void kstat_init_cpu(int cpu) -{ - /* Add any initialisation to kstat here */ - /* Useful when cpu offlining logic is added.. */ -} - -static int __devinit kstat_cpu_notify(struct notifier_block *self, - unsigned long action, void *hcpu) -{ - int cpu = (unsigned long)hcpu; - switch(action) { - case CPU_UP_PREPARE: - kstat_init_cpu(cpu); - break; - default: - break; - } - return NOTIFY_OK; -} - -static struct notifier_block __devinitdata kstat_nb = { - .notifier_call = kstat_cpu_notify, - .next = NULL, -}; - -__init static void init_kstat(void) { - kstat_cpu_notify(&kstat_nb, (unsigned long)CPU_UP_PREPARE, - (void *)(long)smp_processor_id()); - register_cpu_notifier(&kstat_nb); -} - -void __init sched_init(void) -{ - runqueue_t *rq; - int i, j, k; - - /* Init the kstat counters */ - init_kstat(); - for (i = 0; i < NR_CPUS; i++) { - prio_array_t *array; - - rq = cpu_rq(i); - rq->active = rq->arrays; - rq->expired = rq->arrays + 1; - spin_lock_init(&rq->lock); - INIT_LIST_HEAD(&rq->migration_queue); - atomic_set(&rq->nr_iowait, 0); - nr_running_init(rq); - - for (j = 0; j < 2; j++) { - array = rq->arrays + j; - for (k = 0; k < MAX_PRIO; k++) { - INIT_LIST_HEAD(array->queue + k); - __clear_bit(k, array->bitmap); - } - // delimiter for bitsearch - __set_bit(MAX_PRIO, array->bitmap); - } - } - /* - * We have to do a little magic to get the first - * thread right in SMP mode. - */ - rq = this_rq(); - rq->curr = current; - rq->idle = current; - set_task_cpu(current, smp_processor_id()); - wake_up_forked_process(current); - - init_timers(); - - /* - * The boot idle thread does lazy MMU switching as well: - */ - atomic_inc(&init_mm.mm_count); - enter_lazy_tlb(&init_mm, current); -} - -#ifdef CONFIG_DEBUG_SPINLOCK_SLEEP -void __might_sleep(char *file, int line) -{ -#if defined(in_atomic) - static unsigned long prev_jiffy; /* ratelimiting */ - - if ((in_atomic() || irqs_disabled()) && system_running) { - if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy) - return; - prev_jiffy = jiffies; - printk(KERN_ERR "Debug: sleeping function called from invalid" - " context at %s:%d\n", file, line); - printk("in_atomic():%d, irqs_disabled():%d\n", - in_atomic(), irqs_disabled()); - dump_stack(); - } -#endif -} -EXPORT_SYMBOL(__might_sleep); -#endif - - -#if defined(CONFIG_SMP) && defined(CONFIG_PREEMPT) -/* - * This could be a long-held lock. If another CPU holds it for a long time, - * and that CPU is not asked to reschedule then *this* CPU will spin on the - * lock for a long time, even if *this* CPU is asked to reschedule. - * - * So what we do here, in the slow (contended) path is to spin on the lock by - * hand while permitting preemption. - * - * Called inside preempt_disable(). - */ -void __preempt_spin_lock(spinlock_t *lock) -{ - if (preempt_count() > 1) { - _raw_spin_lock(lock); - return; - } - do { - preempt_enable(); - while (spin_is_locked(lock)) - cpu_relax(); - preempt_disable(); - } while (!_raw_spin_trylock(lock)); -} - -EXPORT_SYMBOL(__preempt_spin_lock); - -void __preempt_write_lock(rwlock_t *lock) -{ - if (preempt_count() > 1) { - _raw_write_lock(lock); - return; - } - - do { - preempt_enable(); - while (rwlock_is_locked(lock)) - cpu_relax(); - preempt_disable(); - } while (!_raw_write_trylock(lock)); -} - -EXPORT_SYMBOL(__preempt_write_lock); -#endif /* defined(CONFIG_SMP) && defined(CONFIG_PREEMPT) */ diff -Nur --exclude=RCS --exclude=CVS --exclude=SCCS --exclude=BitKeeper --exclude=ChangeSet linux-2.5-import/kernel/sched.c-new~ linux-2.5-usrdrivers/kernel/sched.c-new~ --- linux-2.5-import/kernel/sched.c-new~ Thu Nov 6 11:37:11 2003 +++ linux-2.5-usrdrivers/kernel/sched.c-new~ Thu Jan 1 10:00:00 1970 @@ -1,2907 +0,0 @@ -/* - * kernel/sched.c - * - * Kernel scheduler and related syscalls - * - * Copyright (C) 1991-2002 Linus Torvalds - * - * 1996-12-23 Modified by Dave Grothe to fix bugs in semaphores and - * make semaphores SMP safe - * 1998-11-19 Implemented schedule_timeout() and related stuff - * by Andrea Arcangeli - * 2002-01-04 New ultra-scalable O(1) scheduler by Ingo Molnar: - * hybrid priority-list and round-robin design with - * an array-switch method of distributing timeslices - * and per-CPU runqueues. Cleanups and useful suggestions - * by Davide Libenzi, preemptible kernel bits by Robert Love. - * 2003-09-03 Interactivity tuning by Con Kolivas. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef CONFIG_NUMA -#define cpu_to_node_mask(cpu) node_to_cpumask(cpu_to_node(cpu)) -#else -#define cpu_to_node_mask(cpu) (cpu_online_map) -#endif - -/* - * Convert user-nice values [ -20 ... 0 ... 19 ] - * to static priority [ MAX_RT_PRIO..MAX_PRIO-1 ], - * and back. - */ -#define NICE_TO_PRIO(nice) (MAX_RT_PRIO + (nice) + 20) -#define PRIO_TO_NICE(prio) ((prio) - MAX_RT_PRIO - 20) -#define TASK_NICE(p) PRIO_TO_NICE((p)->static_prio) - -/* - * 'User priority' is the nice value converted to something we - * can work with better when scaling various scheduler parameters, - * it's a [ 0 ... 39 ] range. - */ -#define USER_PRIO(p) ((p)-MAX_RT_PRIO) -#define TASK_USER_PRIO(p) USER_PRIO((p)->static_prio) -#define MAX_USER_PRIO (USER_PRIO(MAX_PRIO)) -#define AVG_TIMESLICE (MIN_TIMESLICE + ((MAX_TIMESLICE - MIN_TIMESLICE) *\ - (MAX_PRIO-1-NICE_TO_PRIO(0))/(MAX_USER_PRIO - 1))) - -/* - * Some helpers for converting nanosecond timing to jiffy resolution - */ -#define NS_TO_JIFFIES(TIME) ((TIME) / (1000000000 / HZ)) -#define JIFFIES_TO_NS(TIME) ((TIME) * (1000000000 / HZ)) - -/* - * These are the 'tuning knobs' of the scheduler: - * - * Minimum timeslice is 10 msecs, default timeslice is 100 msecs, - * maximum timeslice is 200 msecs. Timeslices get refilled after - * they expire. - */ -#define MIN_TIMESLICE ( 10 * HZ / 1000) -#define MAX_TIMESLICE (200 * HZ / 1000) -#define ON_RUNQUEUE_WEIGHT 30 -#define CHILD_PENALTY 95 -#define PARENT_PENALTY 100 -#define EXIT_WEIGHT 3 -#define PRIO_BONUS_RATIO 25 -#define MAX_BONUS (MAX_USER_PRIO * PRIO_BONUS_RATIO / 100) -#define INTERACTIVE_DELTA 2 -#define MAX_SLEEP_AVG (AVG_TIMESLICE * MAX_BONUS) -#define STARVATION_LIMIT (MAX_SLEEP_AVG) -#define NS_MAX_SLEEP_AVG (JIFFIES_TO_NS(MAX_SLEEP_AVG)) -#define NODE_THRESHOLD 125 -#define CREDIT_LIMIT 100 - -/* - * If a task is 'interactive' then we reinsert it in the active - * array after it has expired its current timeslice. (it will not - * continue to run immediately, it will still roundrobin with - * other interactive tasks.) - * - * This part scales the interactivity limit depending on niceness. - * - * We scale it linearly, offset by the INTERACTIVE_DELTA delta. - * Here are a few examples of different nice levels: - * - * TASK_INTERACTIVE(-20): [1,1,1,1,1,1,1,1,1,0,0] - * TASK_INTERACTIVE(-10): [1,1,1,1,1,1,1,0,0,0,0] - * TASK_INTERACTIVE( 0): [1,1,1,1,0,0,0,0,0,0,0] - * TASK_INTERACTIVE( 10): [1,1,0,0,0,0,0,0,0,0,0] - * TASK_INTERACTIVE( 19): [0,0,0,0,0,0,0,0,0,0,0] - * - * (the X axis represents the possible -5 ... 0 ... +5 dynamic - * priority range a task can explore, a value of '1' means the - * task is rated interactive.) - * - * Ie. nice +19 tasks can never get 'interactive' enough to be - * reinserted into the active array. And only heavily CPU-hog nice -20 - * tasks will be expired. Default nice 0 tasks are somewhere between, - * it takes some effort for them to get interactive, but it's not - * too hard. - */ - -#define CURRENT_BONUS(p) \ - (NS_TO_JIFFIES((p)->sleep_avg) * MAX_BONUS / \ - MAX_SLEEP_AVG) - -#ifdef CONFIG_SMP -#define TIMESLICE_GRANULARITY(p) (MIN_TIMESLICE * \ - (1 << (((MAX_BONUS - CURRENT_BONUS(p)) ? : 1) - 1)) * \ - num_online_cpus()) -#else -#define TIMESLICE_GRANULARITY(p) (MIN_TIMESLICE * \ - (1 << (((MAX_BONUS - CURRENT_BONUS(p)) ? : 1) - 1))) -#endif - -#define SCALE(v1,v1_max,v2_max) \ - (v1) * (v2_max) / (v1_max) - -#define DELTA(p) \ - (SCALE(TASK_NICE(p), 40, MAX_USER_PRIO*PRIO_BONUS_RATIO/100) + \ - INTERACTIVE_DELTA) - -#define TASK_INTERACTIVE(p) \ - ((p)->prio <= (p)->static_prio - DELTA(p)) - -#define JUST_INTERACTIVE_SLEEP(p) \ - (JIFFIES_TO_NS(MAX_SLEEP_AVG * \ - (MAX_BONUS / 2 + DELTA((p)) + 1) / MAX_BONUS - 1)) - -#define HIGH_CREDIT(p) \ - ((p)->interactive_credit > CREDIT_LIMIT) - -#define LOW_CREDIT(p) \ - ((p)->interactive_credit < -CREDIT_LIMIT) - -#define TASK_PREEMPTS_CURR(p, rq) \ - ((p)->prio < (rq)->curr->prio) - -/* - * BASE_TIMESLICE scales user-nice values [ -20 ... 19 ] - * to time slice values. - * - * The higher a thread's priority, the bigger timeslices - * it gets during one round of execution. But even the lowest - * priority thread gets MIN_TIMESLICE worth of execution time. - * - * task_timeslice() is the interface that is used by the scheduler. - */ - -#define BASE_TIMESLICE(p) (MIN_TIMESLICE + \ - ((MAX_TIMESLICE - MIN_TIMESLICE) * (MAX_PRIO-1-(p)->static_prio)/(MAX_USER_PRIO - 1))) - -static inline unsigned int task_timeslice(task_t *p) -{ - return BASE_TIMESLICE(p); -} - -/* - * These are the runqueue data structures: - */ - -#define BITMAP_SIZE ((((MAX_PRIO+1+7)/8)+sizeof(long)-1)/sizeof(long)) - -typedef struct runqueue runqueue_t; - -struct prio_array { - int nr_active; - unsigned long bitmap[BITMAP_SIZE]; - struct list_head queue[MAX_PRIO]; -}; - -/* - * This is the main, per-CPU runqueue data structure. - * - * Locking rule: those places that want to lock multiple runqueues - * (such as the load balancing or the thread migration code), lock - * acquire operations must be ordered by ascending &runqueue. - */ -struct runqueue { - spinlock_t lock; - unsigned long nr_running, nr_switches, expired_timestamp, - nr_uninterruptible; - task_t *curr, *idle; - struct mm_struct *prev_mm; - prio_array_t *active, *expired, arrays[2]; - int prev_cpu_load[NR_CPUS]; -#ifdef CONFIG_NUMA - atomic_t *node_nr_running; - int prev_node_load[MAX_NUMNODES]; -#endif - task_t *migration_thread; - struct list_head migration_queue; - - atomic_t nr_iowait; -}; - -static DEFINE_PER_CPU(struct runqueue, runqueues); - -#define cpu_rq(cpu) (&per_cpu(runqueues, (cpu))) -#define this_rq() (&__get_cpu_var(runqueues)) -#define task_rq(p) cpu_rq(task_cpu(p)) -#define cpu_curr(cpu) (cpu_rq(cpu)->curr) - -/* - * Default context-switch locking: - */ -#ifndef prepare_arch_switch -# define prepare_arch_switch(rq, next) do { } while(0) -# define finish_arch_switch(rq, next) spin_unlock_irq(&(rq)->lock) -# define task_running(rq, p) ((rq)->curr == (p)) -#endif - -#ifdef CONFIG_NUMA - -/* - * Keep track of running tasks. - */ - -static atomic_t node_nr_running[MAX_NUMNODES] ____cacheline_maxaligned_in_smp = - {[0 ...MAX_NUMNODES-1] = ATOMIC_INIT(0)}; - -static inline void nr_running_init(struct runqueue *rq) -{ - rq->node_nr_running = &node_nr_running[0]; -} - -static inline void nr_running_inc(runqueue_t *rq) -{ - atomic_inc(rq->node_nr_running); - rq->nr_running++; -} - -static inline void nr_running_dec(runqueue_t *rq) -{ - atomic_dec(rq->node_nr_running); - rq->nr_running--; -} - -__init void node_nr_running_init(void) -{ - int i; - - for (i = 0; i < NR_CPUS; i++) { - if (cpu_possible(i)) - cpu_rq(i)->node_nr_running = - &node_nr_running[cpu_to_node(i)]; - } -} - -#else /* !CONFIG_NUMA */ - -# define nr_running_init(rq) do { } while (0) -# define nr_running_inc(rq) do { (rq)->nr_running++; } while (0) -# define nr_running_dec(rq) do { (rq)->nr_running--; } while (0) - -#endif /* CONFIG_NUMA */ - -/* - * task_rq_lock - lock the runqueue a given task resides on and disable - * interrupts. Note the ordering: we can safely lookup the task_rq without - * explicitly disabling preemption. - */ -static inline runqueue_t *task_rq_lock(task_t *p, unsigned long *flags) -{ - struct runqueue *rq; - -repeat_lock_task: - local_irq_save(*flags); - rq = task_rq(p); - spin_lock(&rq->lock); - if (unlikely(rq != task_rq(p))) { - spin_unlock_irqrestore(&rq->lock, *flags); - goto repeat_lock_task; - } - return rq; -} - -static inline void task_rq_unlock(runqueue_t *rq, unsigned long *flags) -{ - spin_unlock_irqrestore(&rq->lock, *flags); -} - -/* - * rq_lock - lock a given runqueue and disable interrupts. - */ -static inline runqueue_t *this_rq_lock(void) -{ - runqueue_t *rq; - - local_irq_disable(); - rq = this_rq(); - spin_lock(&rq->lock); - - return rq; -} - -static inline void rq_unlock(runqueue_t *rq) -{ - spin_unlock_irq(&rq->lock); -} - -/* - * Adding/removing a task to/from a priority array: - */ -static inline void dequeue_task(struct task_struct *p, prio_array_t *array) -{ - array->nr_active--; - list_del(&p->run_list); - if (list_empty(array->queue + p->prio)) - __clear_bit(p->prio, array->bitmap); -} - -static inline void enqueue_task(struct task_struct *p, prio_array_t *array) -{ - list_add_tail(&p->run_list, array->queue + p->prio); - __set_bit(p->prio, array->bitmap); - array->nr_active++; - p->array = array; -} - -/* - * effective_prio - return the priority that is based on the static - * priority but is modified by bonuses/penalties. - * - * We scale the actual sleep average [0 .... MAX_SLEEP_AVG] - * into the -5 ... 0 ... +5 bonus/penalty range. - * - * We use 25% of the full 0...39 priority range so that: - * - * 1) nice +19 interactive tasks do not preempt nice 0 CPU hogs. - * 2) nice -20 CPU hogs do not get preempted by nice 0 tasks. - * - * Both properties are important to certain workloads. - */ -static int effective_prio(task_t *p) -{ - int bonus, prio; - - if (rt_task(p)) - return p->prio; - - bonus = CURRENT_BONUS(p) - MAX_BONUS / 2; - - prio = p->static_prio - bonus; - if (prio < MAX_RT_PRIO) - prio = MAX_RT_PRIO; - if (prio > MAX_PRIO-1) - prio = MAX_PRIO-1; - return prio; -} - -/* - * __activate_task - move a task to the runqueue. - */ -static inline void __activate_task(task_t *p, runqueue_t *rq) -{ - enqueue_task(p, rq->active); - nr_running_inc(rq); -} - -static void recalc_task_prio(task_t *p, unsigned long long now) -{ - unsigned long long __sleep_time = now - p->timestamp; - unsigned long sleep_time; - - if (__sleep_time > NS_MAX_SLEEP_AVG) - sleep_time = NS_MAX_SLEEP_AVG; - else - sleep_time = (unsigned long)__sleep_time; - - if (likely(sleep_time > 0)) { - /* - * User tasks that sleep a long time are categorised as - * idle and will get just interactive status to stay active & - * prevent them suddenly becoming cpu hogs and starving - * other processes. - */ - if (p->mm && p->activated != -1 && - sleep_time > JUST_INTERACTIVE_SLEEP(p)){ - p->sleep_avg = JIFFIES_TO_NS(MAX_SLEEP_AVG - - AVG_TIMESLICE); - if (!HIGH_CREDIT(p)) - p->interactive_credit++; - } else { - /* - * The lower the sleep avg a task has the more - * rapidly it will rise with sleep time. - */ - sleep_time *= (MAX_BONUS - CURRENT_BONUS(p)) ? : 1; - - /* - * Tasks with low interactive_credit are limited to - * one timeslice worth of sleep avg bonus. - */ - if (LOW_CREDIT(p) && - sleep_time > JIFFIES_TO_NS(task_timeslice(p))) - sleep_time = - JIFFIES_TO_NS(task_timeslice(p)); - - /* - * Non high_credit tasks waking from uninterruptible - * sleep are limited in their sleep_avg rise as they - * are likely to be cpu hogs waiting on I/O - */ - if (p->activated == -1 && !HIGH_CREDIT(p) && p->mm){ - if (p->sleep_avg >= JUST_INTERACTIVE_SLEEP(p)) - sleep_time = 0; - else if (p->sleep_avg + sleep_time >= - JUST_INTERACTIVE_SLEEP(p)){ - p->sleep_avg = - JUST_INTERACTIVE_SLEEP(p); - sleep_time = 0; - } - } - - /* - * This code gives a bonus to interactive tasks. - * - * The boost works by updating the 'average sleep time' - * value here, based on ->timestamp. The more time a task - * spends sleeping, the higher the average gets - and the - * higher the priority boost gets as well. - */ - p->sleep_avg += sleep_time; - - if (p->sleep_avg > NS_MAX_SLEEP_AVG){ - p->sleep_avg = NS_MAX_SLEEP_AVG; - if (!HIGH_CREDIT(p)) - p->interactive_credit++; - } - } - } - - p->prio = effective_prio(p); -} - -/* - * activate_task - move a task to the runqueue and do priority recalculation - * - * Update all the scheduling statistics stuff. (sleep average - * calculation, priority modifiers, etc.) - */ -static inline void activate_task(task_t *p, runqueue_t *rq) -{ - unsigned long long now = sched_clock(); - - recalc_task_prio(p, now); - - /* - * This checks to make sure it's not an uninterruptible task - * that is now waking up. - */ - if (!p->activated){ - /* - * Tasks which were woken up by interrupts (ie. hw events) - * are most likely of interactive nature. So we give them - * the credit of extending their sleep time to the period - * of time they spend on the runqueue, waiting for execution - * on a CPU, first time around: - */ - if (in_interrupt()) - p->activated = 2; - else - /* - * Normal first-time wakeups get a credit too for on-runqueue - * time, but it will be weighted down: - */ - p->activated = 1; - } - p->timestamp = now; - - __activate_task(p, rq); -} - -/* - * deactivate_task - remove a task from the runqueue. - */ -static inline void deactivate_task(struct task_struct *p, runqueue_t *rq) -{ - nr_running_dec(rq); - if (p->state == TASK_UNINTERRUPTIBLE) - rq->nr_uninterruptible++; - dequeue_task(p, p->array); - p->array = NULL; -} - -/* - * resched_task - mark a task 'to be rescheduled now'. - * - * On UP this means the setting of the need_resched flag, on SMP it - * might also involve a cross-CPU call to trigger the scheduler on - * the target CPU. - */ -static inline void resched_task(task_t *p) -{ -#ifdef CONFIG_SMP - int need_resched, nrpolling; - - preempt_disable(); - /* minimise the chance of sending an interrupt to poll_idle() */ - nrpolling = test_tsk_thread_flag(p,TIF_POLLING_NRFLAG); - need_resched = test_and_set_tsk_thread_flag(p,TIF_NEED_RESCHED); - nrpolling |= test_tsk_thread_flag(p,TIF_POLLING_NRFLAG); - - if (!need_resched && !nrpolling && (task_cpu(p) != smp_processor_id())) - smp_send_reschedule(task_cpu(p)); - preempt_enable(); -#else - set_tsk_need_resched(p); -#endif -} - -#ifdef CONFIG_SMP - -/* - * wait_task_inactive - wait for a thread to unschedule. - * - * The caller must ensure that the task *will* unschedule sometime soon, - * else this function might spin for a *long* time. This function can't - * be called with interrupts off, or it may introduce deadlock with - * smp_call_function() if an IPI is sent by the same process we are - * waiting to become inactive. - */ -void wait_task_inactive(task_t * p) -{ - unsigned long flags; - runqueue_t *rq; - -repeat: - preempt_disable(); - rq = task_rq(p); - if (unlikely(task_running(rq, p))) { - cpu_relax(); - /* - * enable/disable preemption just to make this - * a preemption point - we are busy-waiting - * anyway. - */ - preempt_enable(); - goto repeat; - } - rq = task_rq_lock(p, &flags); - if (unlikely(task_running(rq, p))) { - task_rq_unlock(rq, &flags); - preempt_enable(); - goto repeat; - } - task_rq_unlock(rq, &flags); - preempt_enable(); -} -#endif - -/*** - * try_to_wake_up - wake up a thread - * @p: the to-be-woken-up thread - * @state: the mask of task states that can be woken - * @sync: do a synchronous wakeup? - * @kick: kick the CPU if the task is already running? - * - * Put it on the run-queue if it's not already there. The "current" - * thread is always on the run-queue (except when the actual - * re-schedule is in progress), and as such you're allowed to do - * the simpler "current->state = TASK_RUNNING" to mark yourself - * runnable without the overhead of this. - * - * returns failure only if the task is already active. - */ -static int try_to_wake_up(task_t * p, unsigned int state, int sync, int kick) -{ - unsigned long flags; - int success = 0; - long old_state; - runqueue_t *rq; - -repeat_lock_task: - rq = task_rq_lock(p, &flags); - old_state = p->state; - if (old_state & state) { - if (!p->array) { - /* - * Fast-migrate the task if it's not running or runnable - * currently. Do not violate hard affinity. - */ - if (unlikely(sync && !task_running(rq, p) && - (task_cpu(p) != smp_processor_id()) && - cpu_isset(smp_processor_id(), p->cpus_allowed))) { - - set_task_cpu(p, smp_processor_id()); - task_rq_unlock(rq, &flags); - goto repeat_lock_task; - } - if (old_state == TASK_UNINTERRUPTIBLE){ - rq->nr_uninterruptible--; - /* - * Tasks on involuntary sleep don't earn - * sleep_avg beyond just interactive state. - */ - p->activated = -1; - } - if (sync) - __activate_task(p, rq); - else { - activate_task(p, rq); - if (TASK_PREEMPTS_CURR(p, rq)) - resched_task(rq->curr); - } - success = 1; - } -#ifdef CONFIG_SMP - else - if (unlikely(kick) && task_running(rq, p) && (task_cpu(p) != smp_processor_id())) - smp_send_reschedule(task_cpu(p)); -#endif - p->state = TASK_RUNNING; - } - task_rq_unlock(rq, &flags); - - return success; -} - -int wake_up_process(task_t * p) -{ - return try_to_wake_up(p, TASK_STOPPED | TASK_INTERRUPTIBLE | TASK_UNINTERRUPTIBLE, 0, 0); -} - -EXPORT_SYMBOL(wake_up_process); - -int wake_up_process_kick(task_t * p) -{ - return try_to_wake_up(p, TASK_STOPPED | TASK_INTERRUPTIBLE | TASK_UNINTERRUPTIBLE, 0, 1); -} - -int wake_up_state(task_t *p, unsigned int state) -{ - return try_to_wake_up(p, state, 0, 0); -} - -/* - * wake_up_forked_process - wake up a freshly forked process. - * - * This function will do some initial scheduler statistics housekeeping - * that must be done for every newly created process. - */ -void wake_up_forked_process(task_t * p) -{ - unsigned long flags; - runqueue_t *rq = task_rq_lock(current, &flags); - - p->state = TASK_RUNNING; - /* - * We decrease the sleep average of forking parents - * and children as well, to keep max-interactive tasks - * from forking tasks that are max-interactive. - */ - current->sleep_avg = JIFFIES_TO_NS(CURRENT_BONUS(current) * - PARENT_PENALTY / 100 * MAX_SLEEP_AVG / MAX_BONUS); - - p->sleep_avg = JIFFIES_TO_NS(CURRENT_BONUS(p) * - CHILD_PENALTY / 100 * MAX_SLEEP_AVG / MAX_BONUS); - - p->interactive_credit = 0; - - p->prio = effective_prio(p); - set_task_cpu(p, smp_processor_id()); - - if (unlikely(!current->array)) - __activate_task(p, rq); - else { - p->prio = current->prio; - list_add_tail(&p->run_list, ¤t->run_list); - p->array = current->array; - p->array->nr_active++; - nr_running_inc(rq); - } - task_rq_unlock(rq, &flags); -} - -/* - * Potentially available exiting-child timeslices are - * retrieved here - this way the parent does not get - * penalized for creating too many threads. - * - * (this cannot be used to 'generate' timeslices - * artificially, because any timeslice recovered here - * was given away by the parent in the first place.) - */ -void sched_exit(task_t * p) -{ - unsigned long flags; - - local_irq_save(flags); - if (p->first_time_slice) { - p->parent->time_slice += p->time_slice; - if (unlikely(p->parent->time_slice > MAX_TIMESLICE)) - p->parent->time_slice = MAX_TIMESLICE; - } - local_irq_restore(flags); - /* - * If the child was a (relative-) CPU hog then decrease - * the sleep_avg of the parent as well. - */ - if (p->sleep_avg < p->parent->sleep_avg) - p->parent->sleep_avg = p->parent->sleep_avg / - (EXIT_WEIGHT + 1) * EXIT_WEIGHT + p->sleep_avg / - (EXIT_WEIGHT + 1); -} - -/** - * finish_task_switch - clean up after a task-switch - * @prev: the thread we just switched away from. - * - * We enter this with the runqueue still locked, and finish_arch_switch() - * will unlock it along with doing any other architecture-specific cleanup - * actions. - * - * Note that we may have delayed dropping an mm in context_switch(). If - * so, we finish that here outside of the runqueue lock. (Doing it - * with the lock held can cause deadlocks; see schedule() for - * details.) - */ -static inline void finish_task_switch(task_t *prev) -{ - runqueue_t *rq = this_rq(); - struct mm_struct *mm = rq->prev_mm; - unsigned long prev_task_flags; - - rq->prev_mm = NULL; - - /* - * A task struct has one reference for the use as "current". - * If a task dies, then it sets TASK_ZOMBIE in tsk->state and calls - * schedule one last time. The schedule call will never return, - * and the scheduled task must drop that reference. - * The test for TASK_ZOMBIE must occur while the runqueue locks are - * still held, otherwise prev could be scheduled on another cpu, die - * there before we look at prev->state, and then the reference would - * be dropped twice. - * Manfred Spraul - */ - prev_task_flags = prev->flags; - finish_arch_switch(rq, prev); - if (mm) - mmdrop(mm); - if (unlikely(prev_task_flags & PF_DEAD)) - put_task_struct(prev); -} - -/** - * schedule_tail - first thing a freshly forked thread must call. - * @prev: the thread we just switched away from. - */ -asmlinkage void schedule_tail(task_t *prev) -{ - finish_task_switch(prev); - - if (current->set_child_tid) - put_user(current->pid, current->set_child_tid); -} - -/* - * context_switch - switch to the new MM and the new - * thread's register state. - */ -static inline task_t * context_switch(runqueue_t *rq, task_t *prev, task_t *next) -{ - struct mm_struct *mm = next->mm; - struct mm_struct *oldmm = prev->active_mm; - - if (unlikely(!mm)) { - next->active_mm = oldmm; - atomic_inc(&oldmm->mm_count); - enter_lazy_tlb(oldmm, next); - } else - switch_mm(oldmm, mm, next); - - if (unlikely(!prev->mm)) { - prev->active_mm = NULL; - WARN_ON(rq->prev_mm); - rq->prev_mm = oldmm; - } - - /* Here we just switch the register state and the stack. */ - switch_to(prev, next, prev); - - return prev; -} - -/* - * nr_running, nr_uninterruptible and nr_context_switches: - * - * externally visible scheduler statistics: current number of runnable - * threads, current number of uninterruptible-sleeping threads, total - * number of context switches performed since bootup. - */ -unsigned long nr_running(void) -{ - unsigned long i, sum = 0; - - for (i = 0; i < NR_CPUS; i++) - sum += cpu_rq(i)->nr_running; - - return sum; -} - -unsigned long nr_uninterruptible(void) -{ - unsigned long i, sum = 0; - - for (i = 0; i < NR_CPUS; i++) { - if (!cpu_online(i)) - continue; - sum += cpu_rq(i)->nr_uninterruptible; - } - return sum; -} - -unsigned long nr_context_switches(void) -{ - unsigned long i, sum = 0; - - for (i = 0; i < NR_CPUS; i++) { - if (!cpu_online(i)) - continue; - sum += cpu_rq(i)->nr_switches; - } - return sum; -} - -unsigned long nr_iowait(void) -{ - unsigned long i, sum = 0; - - for (i = 0; i < NR_CPUS; ++i) { - if (!cpu_online(i)) - continue; - sum += atomic_read(&cpu_rq(i)->nr_iowait); - } - return sum; -} - -/* - * double_rq_lock - safely lock two runqueues - * - * Note this does not disable interrupts like task_rq_lock, - * you need to do so manually before calling. - */ -static inline void double_rq_lock(runqueue_t *rq1, runqueue_t *rq2) -{ - if (rq1 == rq2) - spin_lock(&rq1->lock); - else { - if (rq1 < rq2) { - spin_lock(&rq1->lock); - spin_lock(&rq2->lock); - } else { - spin_lock(&rq2->lock); - spin_lock(&rq1->lock); - } - } -} - -/* - * double_rq_unlock - safely unlock two runqueues - * - * Note this does not restore interrupts like task_rq_unlock, - * you need to do so manually after calling. - */ -static inline void double_rq_unlock(runqueue_t *rq1, runqueue_t *rq2) -{ - spin_unlock(&rq1->lock); - if (rq1 != rq2) - spin_unlock(&rq2->lock); -} - -#ifdef CONFIG_NUMA -/* - * If dest_cpu is allowed for this process, migrate the task to it. - * This is accomplished by forcing the cpu_allowed mask to only - * allow dest_cpu, which will force the cpu onto dest_cpu. Then - * the cpu_allowed mask is restored. - */ -static void sched_migrate_task(task_t *p, int dest_cpu) -{ - cpumask_t old_mask; - - old_mask = p->cpus_allowed; - if (!cpu_isset(dest_cpu, old_mask)) - return; - /* force the process onto the specified CPU */ - set_cpus_allowed(p, cpumask_of_cpu(dest_cpu)); - - /* restore the cpus allowed mask */ - set_cpus_allowed(p, old_mask); -} - -/* - * Find the least loaded CPU. Slightly favor the current CPU by - * setting its runqueue length as the minimum to start. - */ -static int sched_best_cpu(struct task_struct *p) -{ - int i, minload, load, best_cpu, node = 0; - cpumask_t cpumask; - - best_cpu = task_cpu(p); - if (cpu_rq(best_cpu)->nr_running <= 2) - return best_cpu; - - minload = 10000000; - for_each_node_with_cpus(i) { - /* - * Node load is always divided by nr_cpus_node to normalise - * load values in case cpu count differs from node to node. - * We first multiply node_nr_running by 10 to get a little - * better resolution. - */ - load = 10 * atomic_read(&node_nr_running[i]) / nr_cpus_node(i); - if (load < minload) { - minload = load; - node = i; - } - } - - minload = 10000000; - cpumask = node_to_cpumask(node); - for (i = 0; i < NR_CPUS; ++i) { - if (!cpu_isset(i, cpumask)) - continue; - if (cpu_rq(i)->nr_running < minload) { - best_cpu = i; - minload = cpu_rq(i)->nr_running; - } - } - return best_cpu; -} - -void sched_balance_exec(void) -{ - int new_cpu; - - if (numnodes > 1) { - new_cpu = sched_best_cpu(current); - if (new_cpu != smp_processor_id()) - sched_migrate_task(current, new_cpu); - } -} - -/* - * Find the busiest node. All previous node loads contribute with a - * geometrically deccaying weight to the load measure: - * load_{t} = load_{t-1}/2 + nr_node_running_{t} - * This way sudden load peaks are flattened out a bit. - * Node load is divided by nr_cpus_node() in order to compare nodes - * of different cpu count but also [first] multiplied by 10 to - * provide better resolution. - */ -static int find_busiest_node(int this_node) -{ - int i, node = -1, load, this_load, maxload; - - if (!nr_cpus_node(this_node)) - return node; - this_load = maxload = (this_rq()->prev_node_load[this_node] >> 1) - + (10 * atomic_read(&node_nr_running[this_node]) - / nr_cpus_node(this_node)); - this_rq()->prev_node_load[this_node] = this_load; - for_each_node_with_cpus(i) { - if (i == this_node) - continue; - load = (this_rq()->prev_node_load[i] >> 1) - + (10 * atomic_read(&node_nr_running[i]) - / nr_cpus_node(i)); - this_rq()->prev_node_load[i] = load; - if (load > maxload && (100*load > NODE_THRESHOLD*this_load)) { - maxload = load; - node = i; - } - } - return node; -} - -#endif /* CONFIG_NUMA */ - -#ifdef CONFIG_SMP - -/* - * double_lock_balance - lock the busiest runqueue - * - * this_rq is locked already. Recalculate nr_running if we have to - * drop the runqueue lock. - */ -static inline unsigned int double_lock_balance(runqueue_t *this_rq, - runqueue_t *busiest, int this_cpu, int idle, unsigned int nr_running) -{ - if (unlikely(!spin_trylock(&busiest->lock))) { - if (busiest < this_rq) { - spin_unlock(&this_rq->lock); - spin_lock(&busiest->lock); - spin_lock(&this_rq->lock); - /* Need to recalculate nr_running */ - if (idle || (this_rq->nr_running > this_rq->prev_cpu_load[this_cpu])) - nr_running = this_rq->nr_running; - else - nr_running = this_rq->prev_cpu_load[this_cpu]; - } else - spin_lock(&busiest->lock); - } - return nr_running; -} - -/* - * find_busiest_queue - find the busiest runqueue among the cpus in cpumask. - */ -static inline runqueue_t *find_busiest_queue(runqueue_t *this_rq, int this_cpu, int idle, int *imbalance, cpumask_t cpumask) -{ - int nr_running, load, max_load, i; - runqueue_t *busiest, *rq_src; - - /* - * We search all runqueues to find the most busy one. - * We do this lockless to reduce cache-bouncing overhead, - * we re-check the 'best' source CPU later on again, with - * the lock held. - * - * We fend off statistical fluctuations in runqueue lengths by - * saving the runqueue length during the previous load-balancing - * operation and using the smaller one the current and saved lengths. - * If a runqueue is long enough for a longer amount of time then - * we recognize it and pull tasks from it. - * - * The 'current runqueue length' is a statistical maximum variable, - * for that one we take the longer one - to avoid fluctuations in - * the other direction. So for a load-balance to happen it needs - * stable long runqueue on the target CPU and stable short runqueue - * on the local runqueue. - * - * We make an exception if this CPU is about to become idle - in - * that case we are less picky about moving a task across CPUs and - * take what can be taken. - */ - if (idle || (this_rq->nr_running > this_rq->prev_cpu_load[this_cpu])) - nr_running = this_rq->nr_running; - else - nr_running = this_rq->prev_cpu_load[this_cpu]; - - busiest = NULL; - max_load = 1; - for (i = 0; i < NR_CPUS; i++) { - if (!cpu_isset(i, cpumask)) - continue; - - rq_src = cpu_rq(i); - if (idle || (rq_src->nr_running < this_rq->prev_cpu_load[i])) - load = rq_src->nr_running; - else - load = this_rq->prev_cpu_load[i]; - this_rq->prev_cpu_load[i] = rq_src->nr_running; - - if ((load > max_load) && (rq_src != this_rq)) { - busiest = rq_src; - max_load = load; - } - } - - if (likely(!busiest)) - goto out; - - *imbalance = max_load - nr_running; - - /* It needs an at least ~25% imbalance to trigger balancing. */ - if (!idle && ((*imbalance)*4 < max_load)) { - busiest = NULL; - goto out; - } - - nr_running = double_lock_balance(this_rq, busiest, this_cpu, idle, nr_running); - /* - * Make sure nothing changed since we checked the - * runqueue length. - */ - if (busiest->nr_running <= nr_running) { - spin_unlock(&busiest->lock); - busiest = NULL; - } -out: - return busiest; -} - -/* - * pull_task - move a task from a remote runqueue to the local runqueue. - * Both runqueues must be locked. - */ -static inline void pull_task(runqueue_t *src_rq, prio_array_t *src_array, task_t *p, runqueue_t *this_rq, int this_cpu) -{ - dequeue_task(p, src_array); - nr_running_dec(src_rq); - set_task_cpu(p, this_cpu); - nr_running_inc(this_rq); - enqueue_task(p, this_rq->active); - /* - * Note that idle threads have a prio of MAX_PRIO, for this test - * to be always true for them. - */ - if (TASK_PREEMPTS_CURR(p, this_rq)) - set_need_resched(); -} - -/* - * Previously: - * - * #define CAN_MIGRATE_TASK(p,rq,this_cpu) \ - * ((!idle || (NS_TO_JIFFIES(now - (p)->timestamp) > \ - * cache_decay_ticks)) && !task_running(rq, p) && \ - * cpu_isset(this_cpu, (p)->cpus_allowed)) - */ - -static inline int -can_migrate_task(task_t *tsk, runqueue_t *rq, int this_cpu, int idle) -{ - unsigned long delta = sched_clock() - tsk->timestamp; - - if (!idle && (delta <= JIFFIES_TO_NS(cache_decay_ticks))) - return 0; - if (task_running(rq, tsk)) - return 0; - if (!cpu_isset(this_cpu, tsk->cpus_allowed)) - return 0; - return 1; -} - -/* - * Current runqueue is empty, or rebalance tick: if there is an - * inbalance (current runqueue is too short) then pull from - * busiest runqueue(s). - * - * We call this with the current runqueue locked, - * irqs disabled. - */ -static void load_balance(runqueue_t *this_rq, int idle, cpumask_t cpumask) -{ - int imbalance, idx, this_cpu = smp_processor_id(); - runqueue_t *busiest; - prio_array_t *array; - struct list_head *head, *curr; - task_t *tmp; - - busiest = find_busiest_queue(this_rq, this_cpu, idle, &imbalance, cpumask); - if (!busiest) - goto out; - - /* - * We only want to steal a number of tasks equal to 1/2 the imbalance, - * otherwise we'll just shift the imbalance to the new queue: - */ - imbalance /= 2; - - /* - * We first consider expired tasks. Those will likely not be - * executed in the near future, and they are most likely to - * be cache-cold, thus switching CPUs has the least effect - * on them. - */ - if (busiest->expired->nr_active) - array = busiest->expired; - else - array = busiest->active; - -new_array: - /* Start searching at priority 0: */ - idx = 0; -skip_bitmap: - if (!idx) - idx = sched_find_first_bit(array->bitmap); - else - idx = find_next_bit(array->bitmap, MAX_PRIO, idx); - if (idx >= MAX_PRIO) { - if (array == busiest->expired) { - array = busiest->active; - goto new_array; - } - goto out_unlock; - } - - head = array->queue + idx; - curr = head->prev; -skip_queue: - tmp = list_entry(curr, task_t, run_list); - - /* - * We do not migrate tasks that are: - * 1) running (obviously), or - * 2) cannot be migrated to this CPU due to cpus_allowed, or - * 3) are cache-hot on their current CPU. - */ - - curr = curr->prev; - - if (!can_migrate_task(tmp, busiest, this_cpu, idle)) { - if (curr != head) - goto skip_queue; - idx++; - goto skip_bitmap; - } - pull_task(busiest, array, tmp, this_rq, this_cpu); - if (!idle && --imbalance) { - if (curr != head) - goto skip_queue; - idx++; - goto skip_bitmap; - } -out_unlock: - spin_unlock(&busiest->lock); -out: - ; -} - -/* - * One of the idle_cpu_tick() and busy_cpu_tick() functions will - * get called every timer tick, on every CPU. Our balancing action - * frequency and balancing agressivity depends on whether the CPU is - * idle or not. - * - * busy-rebalance every 200 msecs. idle-rebalance every 1 msec. (or on - * systems with HZ=100, every 10 msecs.) - * - * On NUMA, do a node-rebalance every 400 msecs. - */ -#define IDLE_REBALANCE_TICK (HZ/1000 ?: 1) -#define BUSY_REBALANCE_TICK (HZ/5 ?: 1) -#define IDLE_NODE_REBALANCE_TICK (IDLE_REBALANCE_TICK * 5) -#define BUSY_NODE_REBALANCE_TICK (BUSY_REBALANCE_TICK * 2) - -#ifdef CONFIG_NUMA -static void balance_node(runqueue_t *this_rq, int idle, int this_cpu) -{ - int node = find_busiest_node(cpu_to_node(this_cpu)); - - if (node >= 0) { - cpumask_t cpumask = node_to_cpumask(node); - cpu_set(this_cpu, cpumask); - spin_lock(&this_rq->lock); - load_balance(this_rq, idle, cpumask); - spin_unlock(&this_rq->lock); - } -} -#endif - -static void rebalance_tick(runqueue_t *this_rq, int idle) -{ -#ifdef CONFIG_NUMA - int this_cpu = smp_processor_id(); -#endif - unsigned long j = jiffies; - - /* - * First do inter-node rebalancing, then intra-node rebalancing, - * if both events happen in the same tick. The inter-node - * rebalancing does not necessarily have to create a perfect - * balance within the node, since we load-balance the most loaded - * node with the current CPU. (ie. other CPUs in the local node - * are not balanced.) - */ - if (idle) { -#ifdef CONFIG_NUMA - if (!(j % IDLE_NODE_REBALANCE_TICK)) - balance_node(this_rq, idle, this_cpu); -#endif - if (!(j % IDLE_REBALANCE_TICK)) { - spin_lock(&this_rq->lock); - load_balance(this_rq, idle, cpu_to_node_mask(this_cpu)); - spin_unlock(&this_rq->lock); - } - return; - } -#ifdef CONFIG_NUMA - if (!(j % BUSY_NODE_REBALANCE_TICK)) - balance_node(this_rq, idle, this_cpu); -#endif - if (!(j % BUSY_REBALANCE_TICK)) { - spin_lock(&this_rq->lock); - load_balance(this_rq, idle, cpu_to_node_mask(this_cpu)); - spin_unlock(&this_rq->lock); - } -} -#else -/* - * on UP we do not need to balance between CPUs: - */ -static inline void rebalance_tick(runqueue_t *this_rq, int idle) -{ -} -#endif - -DEFINE_PER_CPU(struct kernel_stat, kstat) = { { 0 } }; - -EXPORT_PER_CPU_SYMBOL(kstat); - -/* - * We place interactive tasks back into the active array, if possible. - * - * To guarantee that this does not starve expired tasks we ignore the - * interactivity of a task if the first expired task had to wait more - * than a 'reasonable' amount of time. This deadline timeout is - * load-dependent, as the frequency of array switched decreases with - * increasing number of running tasks: - */ -#define EXPIRED_STARVING(rq) \ - (STARVATION_LIMIT && ((rq)->expired_timestamp && \ - (jiffies - (rq)->expired_timestamp >= \ - STARVATION_LIMIT * ((rq)->nr_running) + 1))) - -/* - * This function gets called by the timer code, with HZ frequency. - * We call it with interrupts disabled. - * - * It also gets called by the fork code, when changing the parent's - * timeslices. - */ -void scheduler_tick(int user_ticks, int sys_ticks) -{ - int cpu = smp_processor_id(); - struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat; - runqueue_t *rq = this_rq(); - task_t *p = current; - - if (rcu_pending(cpu)) - rcu_check_callbacks(cpu, user_ticks); - - /* note: this timer irq context must be accounted for as well */ - if (hardirq_count() - HARDIRQ_OFFSET) { - cpustat->irq += sys_ticks; - sys_ticks = 0; - } else if (softirq_count()) { - cpustat->softirq += sys_ticks; - sys_ticks = 0; - } - - if (p == rq->idle) { - if (atomic_read(&rq->nr_iowait) > 0) - cpustat->iowait += sys_ticks; - else - cpustat->idle += sys_ticks; - rebalance_tick(rq, 1); - return; - } - if (TASK_NICE(p) > 0) - cpustat->nice += user_ticks; - else - cpustat->user += user_ticks; - cpustat->system += sys_ticks; - - /* Task might have expired already, but not scheduled off yet */ - if (p->array != rq->active) { - set_tsk_need_resched(p); - goto out; - } - spin_lock(&rq->lock); - /* - * The task was running during this tick - update the - * time slice counter. Note: we do not update a thread's - * priority until it either goes to sleep or uses up its - * timeslice. This makes it possible for interactive tasks - * to use up their timeslices at their highest priority levels. - */ - if (unlikely(rt_task(p))) { - /* - * RR tasks need a special form of timeslice management. - * FIFO tasks have no timeslices. - */ - if ((p->policy == SCHED_RR) && !--p->time_slice) { - p->time_slice = task_timeslice(p); - p->first_time_slice = 0; - set_tsk_need_resched(p); - - /* put it at the end of the queue: */ - dequeue_task(p, rq->active); - enqueue_task(p, rq->active); - } - goto out_unlock; - } - if (!--p->time_slice) { - dequeue_task(p, rq->active); - set_tsk_need_resched(p); - p->prio = effective_prio(p); - p->time_slice = task_timeslice(p); - p->first_time_slice = 0; - - if (!rq->expired_timestamp) - rq->expired_timestamp = jiffies; - if (!TASK_INTERACTIVE(p) || EXPIRED_STARVING(rq)) { - enqueue_task(p, rq->expired); - } else - enqueue_task(p, rq->active); - } else { - /* - * Prevent a too long timeslice allowing a task to monopolize - * the CPU. We do this by splitting up the timeslice into - * smaller pieces. - * - * Note: this does not mean the task's timeslices expire or - * get lost in any way, they just might be preempted by - * another task of equal priority. (one with higher - * priority would have preempted this task already.) We - * requeue this task to the end of the list on this priority - * level, which is in essence a round-robin of tasks with - * equal priority. - * - * This only applies to tasks in the interactive - * delta range with at least TIMESLICE_GRANULARITY to requeue. - */ - if (TASK_INTERACTIVE(p) && !((task_timeslice(p) - - p->time_slice) % TIMESLICE_GRANULARITY(p)) && - (p->time_slice >= TIMESLICE_GRANULARITY(p)) && - (p->array == rq->active)) { - - dequeue_task(p, rq->active); - set_tsk_need_resched(p); - p->prio = effective_prio(p); - enqueue_task(p, rq->active); - } - } -out_unlock: - spin_unlock(&rq->lock); -out: - rebalance_tick(rq, 0); -} - -void scheduling_functions_start_here(void) { } - -/* - * schedule() is the main scheduler function. - */ -asmlinkage void schedule(void) -{ - task_t *prev, *next; - runqueue_t *rq; - prio_array_t *array; - struct list_head *queue; - unsigned long long now; - unsigned long run_time; - int idx; - - /* - * Test if we are atomic. Since do_exit() needs to call into - * schedule() atomically, we ignore that path for now. - * Otherwise, whine if we are scheduling when we should not be. - */ - if (likely(!(current->state & (TASK_DEAD | TASK_ZOMBIE)))) { - if (unlikely(in_atomic())) { - printk(KERN_ERR "bad: scheduling while atomic!\n"); - dump_stack(); - } - } - -need_resched: - preempt_disable(); - prev = current; - rq = this_rq(); - - release_kernel_lock(prev); - now = sched_clock(); - if (likely(now - prev->timestamp < NS_MAX_SLEEP_AVG)) - run_time = now - prev->timestamp; - else - run_time = NS_MAX_SLEEP_AVG; - - /* - * Tasks with interactive credits get charged less run_time - * at high sleep_avg to delay them losing their interactive - * status - */ - if (HIGH_CREDIT(prev)) - run_time /= (CURRENT_BONUS(prev) ? : 1); - - spin_lock_irq(&rq->lock); - - /* - * if entering off of a kernel preemption go straight - * to picking the next task. - */ - if (unlikely(preempt_count() & PREEMPT_ACTIVE)) - goto pick_next_task; - - switch (prev->state) { - case TASK_INTERRUPTIBLE: - if (unlikely(signal_pending(prev))) { - prev->state = TASK_RUNNING; - break; - } - default: - deactivate_task(prev, rq); - prev->nvcsw++; - break; - case TASK_RUNNING: - prev->nivcsw++; - } -pick_next_task: - if (unlikely(!rq->nr_running)) { -#ifdef CONFIG_SMP - load_balance(rq, 1, cpu_to_node_mask(smp_processor_id())); - if (rq->nr_running) - goto pick_next_task; -#endif - next = rq->idle; - rq->expired_timestamp = 0; - goto switch_tasks; - } - - array = rq->active; - if (unlikely(!array->nr_active)) { - /* - * Switch the active and expired arrays. - */ - rq->active = rq->expired; - rq->expired = array; - array = rq->active; - rq->expired_timestamp = 0; - } - - idx = sched_find_first_bit(array->bitmap); - queue = array->queue + idx; - next = list_entry(queue->next, task_t, run_list); - - if (next->activated > 0) { - unsigned long long delta = now - next->timestamp; - - if (next->activated == 1) - delta = delta * (ON_RUNQUEUE_WEIGHT * 128 / 100) / 128; - - array = next->array; - dequeue_task(next, array); - recalc_task_prio(next, next->timestamp + delta); - enqueue_task(next, array); - } - next->activated = 0; -switch_tasks: - prefetch(next); - clear_tsk_need_resched(prev); - RCU_qsctr(task_cpu(prev))++; - - prev->sleep_avg -= run_time; - if ((long)prev->sleep_avg <= 0){ - prev->sleep_avg = 0; - if (!(HIGH_CREDIT(prev) || LOW_CREDIT(prev))) - prev->interactive_credit--; - } - prev->timestamp = now; - - if (likely(prev != next)) { - next->timestamp = now; - rq->nr_switches++; - rq->curr = next; - - prepare_arch_switch(rq, next); - prev = context_switch(rq, prev, next); - barrier(); - - finish_task_switch(prev); - } else - spin_unlock_irq(&rq->lock); - - reacquire_kernel_lock(current); - preempt_enable_no_resched(); - if (test_thread_flag(TIF_NEED_RESCHED)) - goto need_resched; -} - -EXPORT_SYMBOL(schedule); - -#ifdef CONFIG_PREEMPT -/* - * this is is the entry point to schedule() from in-kernel preemption - * off of preempt_enable. Kernel preemptions off return from interrupt - * occur there and call schedule directly. - */ -asmlinkage void preempt_schedule(void) -{ - struct thread_info *ti = current_thread_info(); - - /* - * If there is a non-zero preempt_count or interrupts are disabled, - * we do not want to preempt the current task. Just return.. - */ - if (unlikely(ti->preempt_count || irqs_disabled())) - return; - -need_resched: - ti->preempt_count = PREEMPT_ACTIVE; - schedule(); - ti->preempt_count = 0; - - /* we could miss a preemption opportunity between schedule and now */ - barrier(); - if (unlikely(test_thread_flag(TIF_NEED_RESCHED))) - goto need_resched; -} - -EXPORT_SYMBOL(preempt_schedule); -#endif /* CONFIG_PREEMPT */ - -int default_wake_function(wait_queue_t *curr, unsigned mode, int sync) -{ - task_t *p = curr->task; - return try_to_wake_up(p, mode, sync, 0); -} - -EXPORT_SYMBOL(default_wake_function); - -/* - * The core wakeup function. Non-exclusive wakeups (nr_exclusive == 0) just - * wake everything up. If it's an exclusive wakeup (nr_exclusive == small +ve - * number) then we wake all the non-exclusive tasks and one exclusive task. - * - * There are circumstances in which we can try to wake a task which has already - * started to run but is not in state TASK_RUNNING. try_to_wake_up() returns - * zero in this (rare) case, and we handle it by continuing to scan the queue. - */ -static void __wake_up_common(wait_queue_head_t *q, unsigned int mode, int nr_exclusive, int sync) -{ - struct list_head *tmp, *next; - - list_for_each_safe(tmp, next, &q->task_list) { - wait_queue_t *curr; - unsigned flags; - curr = list_entry(tmp, wait_queue_t, task_list); - flags = curr->flags; - if (curr->func(curr, mode, sync) && - (flags & WQ_FLAG_EXCLUSIVE) && - !--nr_exclusive) - break; - } -} - -/** - * __wake_up - wake up threads blocked on a waitqueue. - * @q: the waitqueue - * @mode: which threads - * @nr_exclusive: how many wake-one or wake-many threads to wake up - */ -void __wake_up(wait_queue_head_t *q, unsigned int mode, int nr_exclusive) -{ - unsigned long flags; - - spin_lock_irqsave(&q->lock, flags); - __wake_up_common(q, mode, nr_exclusive, 0); - spin_unlock_irqrestore(&q->lock, flags); -} - -EXPORT_SYMBOL(__wake_up); - -/* - * Same as __wake_up but called with the spinlock in wait_queue_head_t held. - */ -void __wake_up_locked(wait_queue_head_t *q, unsigned int mode) -{ - __wake_up_common(q, mode, 1, 0); -} - -/** - * __wake_up - sync- wake up threads blocked on a waitqueue. - * @q: the waitqueue - * @mode: which threads - * @nr_exclusive: how many wake-one or wake-many threads to wake up - * - * The sync wakeup differs that the waker knows that it will schedule - * away soon, so while the target thread will be woken up, it will not - * be migrated to another CPU - ie. the two threads are 'synchronized' - * with each other. This can prevent needless bouncing between CPUs. - * - * On UP it can prevent extra preemption. - */ -void __wake_up_sync(wait_queue_head_t *q, unsigned int mode, int nr_exclusive) -{ - unsigned long flags; - - if (unlikely(!q)) - return; - - spin_lock_irqsave(&q->lock, flags); - if (likely(nr_exclusive)) - __wake_up_common(q, mode, nr_exclusive, 1); - else - __wake_up_common(q, mode, nr_exclusive, 0); - spin_unlock_irqrestore(&q->lock, flags); -} - -EXPORT_SYMBOL_GPL(__wake_up_sync); /* For internal use only */ - -void complete(struct completion *x) -{ - unsigned long flags; - - spin_lock_irqsave(&x->wait.lock, flags); - x->done++; - __wake_up_common(&x->wait, TASK_UNINTERRUPTIBLE | TASK_INTERRUPTIBLE, 1, 0); - spin_unlock_irqrestore(&x->wait.lock, flags); -} - -EXPORT_SYMBOL(complete); - -void complete_all(struct completion *x) -{ - unsigned long flags; - - spin_lock_irqsave(&x->wait.lock, flags); - x->done += UINT_MAX/2; - __wake_up_common(&x->wait, TASK_UNINTERRUPTIBLE | TASK_INTERRUPTIBLE, 0, 0); - spin_unlock_irqrestore(&x->wait.lock, flags); -} - -void wait_for_completion(struct completion *x) -{ - might_sleep(); - spin_lock_irq(&x->wait.lock); - if (!x->done) { - DECLARE_WAITQUEUE(wait, current); - - wait.flags |= WQ_FLAG_EXCLUSIVE; - __add_wait_queue_tail(&x->wait, &wait); - do { - __set_current_state(TASK_UNINTERRUPTIBLE); - spin_unlock_irq(&x->wait.lock); - schedule(); - spin_lock_irq(&x->wait.lock); - } while (!x->done); - __remove_wait_queue(&x->wait, &wait); - } - x->done--; - spin_unlock_irq(&x->wait.lock); -} - -EXPORT_SYMBOL(wait_for_completion); - -#define SLEEP_ON_VAR \ - unsigned long flags; \ - wait_queue_t wait; \ - init_waitqueue_entry(&wait, current); - -#define SLEEP_ON_HEAD \ - spin_lock_irqsave(&q->lock,flags); \ - __add_wait_queue(q, &wait); \ - spin_unlock(&q->lock); - -#define SLEEP_ON_TAIL \ - spin_lock_irq(&q->lock); \ - __remove_wait_queue(q, &wait); \ - spin_unlock_irqrestore(&q->lock, flags); - -void interruptible_sleep_on(wait_queue_head_t *q) -{ - SLEEP_ON_VAR - - current->state = TASK_INTERRUPTIBLE; - - SLEEP_ON_HEAD - schedule(); - SLEEP_ON_TAIL -} - -EXPORT_SYMBOL(interruptible_sleep_on); - -long interruptible_sleep_on_timeout(wait_queue_head_t *q, long timeout) -{ - SLEEP_ON_VAR - - current->state = TASK_INTERRUPTIBLE; - - SLEEP_ON_HEAD - timeout = schedule_timeout(timeout); - SLEEP_ON_TAIL - - return timeout; -} - -EXPORT_SYMBOL(interruptible_sleep_on_timeout); - -void sleep_on(wait_queue_head_t *q) -{ - SLEEP_ON_VAR - - current->state = TASK_UNINTERRUPTIBLE; - - SLEEP_ON_HEAD - schedule(); - SLEEP_ON_TAIL -} - -EXPORT_SYMBOL(sleep_on); - -long sleep_on_timeout(wait_queue_head_t *q, long timeout) -{ - SLEEP_ON_VAR - - current->state = TASK_UNINTERRUPTIBLE; - - SLEEP_ON_HEAD - timeout = schedule_timeout(timeout); - SLEEP_ON_TAIL - - return timeout; -} - -EXPORT_SYMBOL(sleep_on_timeout); - -void scheduling_functions_end_here(void) { } - -void set_user_nice(task_t *p, long nice) -{ - unsigned long flags; - prio_array_t *array; - runqueue_t *rq; - int old_prio, new_prio, delta; - - if (TASK_NICE(p) == nice || nice < -20 || nice > 19) - return; - /* - * We have to be careful, if called from sys_setpriority(), - * the task might be in the middle of scheduling on another CPU. - */ - rq = task_rq_lock(p, &flags); - /* - * The RT priorities are set via setscheduler(), but we still - * allow the 'normal' nice value to be set - but as expected - * it wont have any effect on scheduling until the task is - * not SCHED_NORMAL: - */ - if (rt_task(p)) { - p->static_prio = NICE_TO_PRIO(nice); - goto out_unlock; - } - array = p->array; - if (array) - dequeue_task(p, array); - - old_prio = p->prio; - new_prio = NICE_TO_PRIO(nice); - delta = new_prio - old_prio; - p->static_prio = NICE_TO_PRIO(nice); - p->prio += delta; - - if (array) { - enqueue_task(p, array); - /* - * If the task increased its priority or is running and - * lowered its priority, then reschedule its CPU: - */ - if (delta < 0 || (delta > 0 && task_running(rq, p))) - resched_task(rq->curr); - } -out_unlock: - task_rq_unlock(rq, &flags); -} - -EXPORT_SYMBOL(set_user_nice); - -#ifndef __alpha__ - -/* - * sys_nice - change the priority of the current process. - * @increment: priority increment - * - * sys_setpriority is a more generic, but much slower function that - * does similar things. - */ -asmlinkage long sys_nice(int increment) -{ - int retval; - long nice; - - /* - * Setpriority might change our priority at the same moment. - * We don't have to worry. Conceptually one call occurs first - * and we have a single winner. - */ - if (increment < 0) { - if (!capable(CAP_SYS_NICE)) - return -EPERM; - if (increment < -40) - increment = -40; - } - if (increment > 40) - increment = 40; - - nice = PRIO_TO_NICE(current->static_prio) + increment; - if (nice < -20) - nice = -20; - if (nice > 19) - nice = 19; - - retval = security_task_setnice(current, nice); - if (retval) - return retval; - - set_user_nice(current, nice); - return 0; -} - -#endif - -/** - * task_prio - return the priority value of a given task. - * @p: the task in question. - * - * This is the priority value as seen by users in /proc. - * RT tasks are offset by -200. Normal tasks are centered - * around 0, value goes from -16 to +15. - */ -int task_prio(task_t *p) -{ - return p->prio - MAX_RT_PRIO; -} - -/** - * task_nice - return the nice value of a given task. - * @p: the task in question. - */ -int task_nice(task_t *p) -{ - return TASK_NICE(p); -} - -EXPORT_SYMBOL(task_nice); - -/** - * task_curr - is this task currently executing on a CPU? - * @p: the task in question. - */ -int task_curr(task_t *p) -{ - return cpu_curr(task_cpu(p)) == p; -} - -/** - * idle_cpu - is a given cpu idle currently? - * @cpu: the processor in question. - */ -int idle_cpu(int cpu) -{ - return cpu_curr(cpu) == cpu_rq(cpu)->idle; -} - -EXPORT_SYMBOL_GPL(idle_cpu); - -/** - * find_process_by_pid - find a process with a matching PID value. - * @pid: the pid in question. - */ -static inline task_t *find_process_by_pid(pid_t pid) -{ - return pid ? find_task_by_pid(pid) : current; -} - -/* - * setscheduler - change the scheduling policy and/or RT priority of a thread. - */ -static int setscheduler(pid_t pid, int policy, struct sched_param __user *param) -{ - struct sched_param lp; - int retval = -EINVAL; - int oldprio; - prio_array_t *array; - unsigned long flags; - runqueue_t *rq; - task_t *p; - - if (!param || pid < 0) - goto out_nounlock; - - retval = -EFAULT; - if (copy_from_user(&lp, param, sizeof(struct sched_param))) - goto out_nounlock; - - /* - * We play safe to avoid deadlocks. - */ - read_lock_irq(&tasklist_lock); - - p = find_process_by_pid(pid); - - retval = -ESRCH; - if (!p) - goto out_unlock_tasklist; - - /* - * To be able to change p->policy safely, the apropriate - * runqueue lock must be held. - */ - rq = task_rq_lock(p, &flags); - - if (policy < 0) - policy = p->policy; - else { - retval = -EINVAL; - if (policy != SCHED_FIFO && policy != SCHED_RR && - policy != SCHED_NORMAL) - goto out_unlock; - } - - /* - * Valid priorities for SCHED_FIFO and SCHED_RR are - * 1..MAX_USER_RT_PRIO-1, valid priority for SCHED_NORMAL is 0. - */ - retval = -EINVAL; - if (lp.sched_priority < 0 || lp.sched_priority > MAX_USER_RT_PRIO-1) - goto out_unlock; - if ((policy == SCHED_NORMAL) != (lp.sched_priority == 0)) - goto out_unlock; - - retval = -EPERM; - if ((policy == SCHED_FIFO || policy == SCHED_RR) && - !capable(CAP_SYS_NICE)) - goto out_unlock; - if ((current->euid != p->euid) && (current->euid != p->uid) && - !capable(CAP_SYS_NICE)) - goto out_unlock; - - retval = security_task_setscheduler(p, policy, &lp); - if (retval) - goto out_unlock; - - array = p->array; - if (array) - deactivate_task(p, task_rq(p)); - retval = 0; - p->policy = policy; - p->rt_priority = lp.sched_priority; - oldprio = p->prio; - if (policy != SCHED_NORMAL) - p->prio = MAX_USER_RT_PRIO-1 - p->rt_priority; - else - p->prio = p->static_prio; - if (array) { - __activate_task(p, task_rq(p)); - /* - * Reschedule if we are currently running on this runqueue and - * our priority decreased, or if we are not currently running on - * this runqueue and our priority is higher than the current's - */ - if (rq->curr == p) { - if (p->prio > oldprio) - resched_task(rq->curr); - } else if (p->prio < rq->curr->prio) - resched_task(rq->curr); - } - -out_unlock: - task_rq_unlock(rq, &flags); -out_unlock_tasklist: - read_unlock_irq(&tasklist_lock); - -out_nounlock: - return retval; -} - -/** - * sys_sched_setscheduler - set/change the scheduler policy and RT priority - * @pid: the pid in question. - * @policy: new policy - * @param: structure containing the new RT priority. - */ -asmlinkage long sys_sched_setscheduler(pid_t pid, int policy, - struct sched_param __user *param) -{ - return setscheduler(pid, policy, param); -} - -/** - * sys_sched_setparam - set/change the RT priority of a thread - * @pid: the pid in question. - * @param: structure containing the new RT priority. - */ -asmlinkage long sys_sched_setparam(pid_t pid, struct sched_param __user *param) -{ - return setscheduler(pid, -1, param); -} - -/** - * sys_sched_getscheduler - get the policy (scheduling class) of a thread - * @pid: the pid in question. - */ -asmlinkage long sys_sched_getscheduler(pid_t pid) -{ - int retval = -EINVAL; - task_t *p; - - if (pid < 0) - goto out_nounlock; - - retval = -ESRCH; - read_lock(&tasklist_lock); - p = find_process_by_pid(pid); - if (p) { - retval = security_task_getscheduler(p); - if (!retval) - retval = p->policy; - } - read_unlock(&tasklist_lock); - -out_nounlock: - return retval; -} - -/** - * sys_sched_getscheduler - get the RT priority of a thread - * @pid: the pid in question. - * @param: structure containing the RT priority. - */ -asmlinkage long sys_sched_getparam(pid_t pid, struct sched_param __user *param) -{ - struct sched_param lp; - int retval = -EINVAL; - task_t *p; - - if (!param || pid < 0) - goto out_nounlock; - - read_lock(&tasklist_lock); - p = find_process_by_pid(pid); - retval = -ESRCH; - if (!p) - goto out_unlock; - - retval = security_task_getscheduler(p); - if (retval) - goto out_unlock; - - lp.sched_priority = p->rt_priority; - read_unlock(&tasklist_lock); - - /* - * This one might sleep, we cannot do it with a spinlock held ... - */ - retval = copy_to_user(param, &lp, sizeof(*param)) ? -EFAULT : 0; - -out_nounlock: - return retval; - -out_unlock: - read_unlock(&tasklist_lock); - return retval; -} - -/** - * sys_sched_setaffinity - set the cpu affinity of a process - * @pid: pid of the process - * @len: length in bytes of the bitmask pointed to by user_mask_ptr - * @user_mask_ptr: user-space pointer to the new cpu mask - */ -asmlinkage long sys_sched_setaffinity(pid_t pid, unsigned int len, - unsigned long __user *user_mask_ptr) -{ - cpumask_t new_mask; - int retval; - task_t *p; - - if (len < sizeof(new_mask)) - return -EINVAL; - - if (copy_from_user(&new_mask, user_mask_ptr, sizeof(new_mask))) - return -EFAULT; - - read_lock(&tasklist_lock); - - p = find_process_by_pid(pid); - if (!p) { - read_unlock(&tasklist_lock); - return -ESRCH; - } - - /* - * It is not safe to call set_cpus_allowed with the - * tasklist_lock held. We will bump the task_struct's - * usage count and then drop tasklist_lock. - */ - get_task_struct(p); - read_unlock(&tasklist_lock); - - retval = -EPERM; - if ((current->euid != p->euid) && (current->euid != p->uid) && - !capable(CAP_SYS_NICE)) - goto out_unlock; - - retval = set_cpus_allowed(p, new_mask); - -out_unlock: - put_task_struct(p); - return retval; -} - -/** - * sys_sched_getaffinity - get the cpu affinity of a process - * @pid: pid of the process - * @len: length in bytes of the bitmask pointed to by user_mask_ptr - * @user_mask_ptr: user-space pointer to hold the current cpu mask - */ -asmlinkage long sys_sched_getaffinity(pid_t pid, unsigned int len, - unsigned long __user *user_mask_ptr) -{ - unsigned int real_len; - cpumask_t mask; - int retval; - task_t *p; - - real_len = sizeof(mask); - if (len < real_len) - return -EINVAL; - - read_lock(&tasklist_lock); - - retval = -ESRCH; - p = find_process_by_pid(pid); - if (!p) - goto out_unlock; - - retval = 0; - cpus_and(mask, p->cpus_allowed, cpu_online_map); - -out_unlock: - read_unlock(&tasklist_lock); - if (retval) - return retval; - if (copy_to_user(user_mask_ptr, &mask, real_len)) - return -EFAULT; - return real_len; -} - -/** - * sys_sched_yield - yield the current processor to other threads. - * - * this function yields the current CPU by moving the calling thread - * to the expired array. If there are no other threads running on this - * CPU then this function will return. - */ -asmlinkage long sys_sched_yield(void) -{ - runqueue_t *rq = this_rq_lock(); - prio_array_t *array = current->array; - - /* - * We implement yielding by moving the task into the expired - * queue. - * - * (special rule: RT tasks will just roundrobin in the active - * array.) - */ - if (likely(!rt_task(current))) { - dequeue_task(current, array); - enqueue_task(current, rq->expired); - } else { - list_del(¤t->run_list); - list_add_tail(¤t->run_list, array->queue + current->prio); - } - /* - * Since we are going to call schedule() anyway, there's - * no need to preempt: - */ - _raw_spin_unlock(&rq->lock); - preempt_enable_no_resched(); - - schedule(); - - return 0; -} - -void __cond_resched(void) -{ - set_current_state(TASK_RUNNING); - schedule(); -} - -EXPORT_SYMBOL(__cond_resched); - -/** - * yield - yield the current processor to other threads. - * - * this is a shortcut for kernel-space yielding - it marks the - * thread runnable and calls sys_sched_yield(). - */ -void yield(void) -{ - set_current_state(TASK_RUNNING); - sys_sched_yield(); -} - -EXPORT_SYMBOL(yield); - -/* - * This task is about to go to sleep on IO. Increment rq->nr_iowait so - * that process accounting knows that this is a task in IO wait state. - * - * But don't do that if it is a deliberate, throttling IO wait (this task - * has set its backing_dev_info: the queue against which it should throttle) - */ -void io_schedule(void) -{ - struct runqueue *rq = this_rq(); - - atomic_inc(&rq->nr_iowait); - schedule(); - atomic_dec(&rq->nr_iowait); -} - -EXPORT_SYMBOL(io_schedule); - -long io_schedule_timeout(long timeout) -{ - struct runqueue *rq = this_rq(); - long ret; - - atomic_inc(&rq->nr_iowait); - ret = schedule_timeout(timeout); - atomic_dec(&rq->nr_iowait); - return ret; -} - -/** - * sys_sched_get_priority_max - return maximum RT priority. - * @policy: scheduling class. - * - * this syscall returns the maximum rt_priority that can be used - * by a given scheduling class. - */ -asmlinkage long sys_sched_get_priority_max(int policy) -{ - int ret = -EINVAL; - - switch (policy) { - case SCHED_FIFO: - case SCHED_RR: - ret = MAX_USER_RT_PRIO-1; - break; - case SCHED_NORMAL: - ret = 0; - break; - } - return ret; -} - -/** - * sys_sched_get_priority_min - return minimum RT priority. - * @policy: scheduling class. - * - * this syscall returns the minimum rt_priority that can be used - * by a given scheduling class. - */ -asmlinkage long sys_sched_get_priority_min(int policy) -{ - int ret = -EINVAL; - - switch (policy) { - case SCHED_FIFO: - case SCHED_RR: - ret = 1; - break; - case SCHED_NORMAL: - ret = 0; - } - return ret; -} - -/** - * sys_sched_rr_get_interval - return the default timeslice of a process. - * @pid: pid of the process. - * @interval: userspace pointer to the timeslice value. - * - * this syscall writes the default timeslice value of a given process - * into the user-space timespec buffer. A value of '0' means infinity. - */ -asmlinkage long sys_sched_rr_get_interval(pid_t pid, struct timespec __user *interval) -{ - int retval = -EINVAL; - struct timespec t; - task_t *p; - - if (pid < 0) - goto out_nounlock; - - retval = -ESRCH; - read_lock(&tasklist_lock); - p = find_process_by_pid(pid); - if (!p) - goto out_unlock; - - retval = security_task_getscheduler(p); - if (retval) - goto out_unlock; - - jiffies_to_timespec(p->policy & SCHED_FIFO ? - 0 : task_timeslice(p), &t); - read_unlock(&tasklist_lock); - retval = copy_to_user(interval, &t, sizeof(t)) ? -EFAULT : 0; -out_nounlock: - return retval; -out_unlock: - read_unlock(&tasklist_lock); - return retval; -} - -static inline struct task_struct *eldest_child(struct task_struct *p) -{ - if (list_empty(&p->children)) return NULL; - return list_entry(p->children.next,struct task_struct,sibling); -} - -static inline struct task_struct *older_sibling(struct task_struct *p) -{ - if (p->sibling.prev==&p->parent->children) return NULL; - return list_entry(p->sibling.prev,struct task_struct,sibling); -} - -static inline struct task_struct *younger_sibling(struct task_struct *p) -{ - if (p->sibling.next==&p->parent->children) return NULL; - return list_entry(p->sibling.next,struct task_struct,sibling); -} - -static void show_task(task_t * p) -{ - unsigned long free = 0; - task_t *relative; - int state; - static const char * stat_nam[] = { "R", "S", "D", "T", "Z", "W" }; - - printk("%-13.13s ", p->comm); - state = p->state ? __ffs(p->state) + 1 : 0; - if (((unsigned) state) < sizeof(stat_nam)/sizeof(char *)) - printk(stat_nam[state]); - else - printk(" "); -#if (BITS_PER_LONG == 32) - if (p == current) - printk(" current "); - else - printk(" %08lX ", thread_saved_pc(p)); -#else - if (p == current) - printk(" current task "); - else - printk(" %016lx ", thread_saved_pc(p)); -#endif - { - unsigned long * n = (unsigned long *) (p->thread_info+1); - while (!*n) - n++; - free = (unsigned long) n - (unsigned long)(p->thread_info+1); - } - printk("%5lu %5d %6d ", free, p->pid, p->parent->pid); - if ((relative = eldest_child(p))) - printk("%5d ", relative->pid); - else - printk(" "); - if ((relative = younger_sibling(p))) - printk("%7d", relative->pid); - else - printk(" "); - if ((relative = older_sibling(p))) - printk(" %5d", relative->pid); - else - printk(" "); - if (!p->mm) - printk(" (L-TLB)\n"); - else - printk(" (NOTLB)\n"); - - show_stack(p, NULL); -} - -void show_state(void) -{ - task_t *g, *p; - -#if (BITS_PER_LONG == 32) - printk("\n" - " free sibling\n"); - printk(" task PC stack pid father child younger older\n"); -#else - printk("\n" - " free sibling\n"); - printk(" task PC stack pid father child younger older\n"); -#endif - read_lock(&tasklist_lock); - do_each_thread(g, p) { - /* - * reset the NMI-timeout, listing all files on a slow - * console might take alot of time: - */ - touch_nmi_watchdog(); - show_task(p); - } while_each_thread(g, p); - - read_unlock(&tasklist_lock); -} - -void __init init_idle(task_t *idle, int cpu) -{ - runqueue_t *idle_rq = cpu_rq(cpu), *rq = cpu_rq(task_cpu(idle)); - unsigned long flags; - - local_irq_save(flags); - double_rq_lock(idle_rq, rq); - - idle_rq->curr = idle_rq->idle = idle; - deactivate_task(idle, rq); - idle->array = NULL; - idle->prio = MAX_PRIO; - idle->state = TASK_RUNNING; - set_task_cpu(idle, cpu); - double_rq_unlock(idle_rq, rq); - set_tsk_need_resched(idle); - local_irq_restore(flags); - - /* Set the preempt count _outside_ the spinlocks! */ -#ifdef CONFIG_PREEMPT - idle->thread_info->preempt_count = (idle->lock_depth >= 0); -#else - idle->thread_info->preempt_count = 0; -#endif -} - -#ifdef CONFIG_SMP -/* - * This is how migration works: - * - * 1) we queue a migration_req_t structure in the source CPU's - * runqueue and wake up that CPU's migration thread. - * 2) we down() the locked semaphore => thread blocks. - * 3) migration thread wakes up (implicitly it forces the migrated - * thread off the CPU) - * 4) it gets the migration request and checks whether the migrated - * task is still in the wrong runqueue. - * 5) if it's in the wrong runqueue then the migration thread removes - * it and puts it into the right queue. - * 6) migration thread up()s the semaphore. - * 7) we wake up and the migration is done. - */ - -typedef struct { - struct list_head list; - task_t *task; - struct completion done; -} migration_req_t; - -/* - * Change a given task's CPU affinity. Migrate the thread to a - * proper CPU and schedule it away if the CPU it's executing on - * is removed from the allowed bitmask. - * - * NOTE: the caller must have a valid reference to the task, the - * task must not exit() & deallocate itself prematurely. The - * call is not atomic; no spinlocks may be held. - */ -int set_cpus_allowed(task_t *p, cpumask_t new_mask) -{ - unsigned long flags; - migration_req_t req; - runqueue_t *rq; - - if (any_online_cpu(new_mask) == NR_CPUS) - return -EINVAL; - - rq = task_rq_lock(p, &flags); - p->cpus_allowed = new_mask; - /* - * Can the task run on the task's current CPU? If not then - * migrate the thread off to a proper CPU. - */ - if (cpu_isset(task_cpu(p), new_mask)) { - task_rq_unlock(rq, &flags); - return 0; - } - /* - * If the task is not on a runqueue (and not running), then - * it is sufficient to simply update the task's cpu field. - */ - if (!p->array && !task_running(rq, p)) { - set_task_cpu(p, any_online_cpu(p->cpus_allowed)); - task_rq_unlock(rq, &flags); - return 0; - } - init_completion(&req.done); - req.task = p; - list_add(&req.list, &rq->migration_queue); - task_rq_unlock(rq, &flags); - - wake_up_process(rq->migration_thread); - - wait_for_completion(&req.done); - return 0; -} - -EXPORT_SYMBOL_GPL(set_cpus_allowed); - -/* Move (not current) task off this cpu, onto dest cpu. */ -static void move_task_away(struct task_struct *p, int dest_cpu) -{ - runqueue_t *rq_dest; - unsigned long flags; - - rq_dest = cpu_rq(dest_cpu); - - local_irq_save(flags); - double_rq_lock(this_rq(), rq_dest); - if (task_cpu(p) != smp_processor_id()) - goto out; /* Already moved */ - - set_task_cpu(p, dest_cpu); - if (p->array) { - deactivate_task(p, this_rq()); - activate_task(p, rq_dest); - if (p->prio < rq_dest->curr->prio) - resched_task(rq_dest->curr); - } - out: - double_rq_unlock(this_rq(), rq_dest); - local_irq_restore(flags); -} - -typedef struct { - int cpu; - struct completion startup_done; - task_t *task; -} migration_startup_t; - -/* - * migration_thread - this is a highprio system thread that performs - * thread migration by bumping thread off CPU then 'pushing' onto - * another runqueue. - */ -static int migration_thread(void * data) -{ - /* Marking "param" __user is ok, since we do a set_fs(KERNEL_DS); */ - struct sched_param __user param = { .sched_priority = MAX_RT_PRIO-1 }; - migration_startup_t *startup = data; - int cpu = startup->cpu; - runqueue_t *rq; - int ret; - - startup->task = current; - complete(&startup->startup_done); - set_current_state(TASK_UNINTERRUPTIBLE); - schedule(); - - BUG_ON(smp_processor_id() != cpu); - - daemonize("migration/%d", cpu); - set_fs(KERNEL_DS); - - ret = setscheduler(0, SCHED_FIFO, ¶m); - - rq = this_rq(); - rq->migration_thread = current; - - for (;;) { - struct list_head *head; - migration_req_t *req; - - if (current->flags & PF_FREEZE) - refrigerator(PF_IOTHREAD); - - spin_lock_irq(&rq->lock); - head = &rq->migration_queue; - current->state = TASK_INTERRUPTIBLE; - if (list_empty(head)) { - spin_unlock_irq(&rq->lock); - schedule(); - continue; - } - req = list_entry(head->next, migration_req_t, list); - list_del_init(head->next); - spin_unlock_irq(&rq->lock); - - move_task_away(req->task, - any_online_cpu(req->task->cpus_allowed)); - complete(&req->done); - } -} - -/* - * migration_call - callback that gets triggered when a CPU is added. - * Here we can start up the necessary migration thread for the new CPU. - */ -static int migration_call(struct notifier_block *nfb, - unsigned long action, - void *hcpu) -{ - long cpu = (long) hcpu; - migration_startup_t startup; - - switch (action) { - case CPU_ONLINE: - - printk("Starting migration thread for cpu %li\n", cpu); - - startup.cpu = cpu; - startup.task = NULL; - init_completion(&startup.startup_done); - - kernel_thread(migration_thread, &startup, CLONE_KERNEL); - wait_for_completion(&startup.startup_done); - wait_task_inactive(startup.task); - - startup.task->thread_info->cpu = cpu; - startup.task->cpus_allowed = cpumask_of_cpu(cpu); - - wake_up_process(startup.task); - - while (!cpu_rq(cpu)->migration_thread) - yield(); - - break; - } - return NOTIFY_OK; -} - -static struct notifier_block migration_notifier = { &migration_call, NULL, 0 }; - -__init int migration_init(void) -{ - /* Start one for boot CPU. */ - migration_call(&migration_notifier, CPU_ONLINE, - (void *)(long)smp_processor_id()); - register_cpu_notifier(&migration_notifier); - return 0; -} - -#endif - -#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT) -/* - * The 'big kernel lock' - * - * This spinlock is taken and released recursively by lock_kernel() - * and unlock_kernel(). It is transparently dropped and reaquired - * over schedule(). It is used to protect legacy code that hasn't - * been migrated to a proper locking design yet. - * - * Don't use in new code. - */ -spinlock_t kernel_flag __cacheline_aligned_in_smp = SPIN_LOCK_UNLOCKED; - -EXPORT_SYMBOL(kernel_flag); -#endif - -static void kstat_init_cpu(int cpu) -{ - /* Add any initialisation to kstat here */ - /* Useful when cpu offlining logic is added.. */ -} - -static int __devinit kstat_cpu_notify(struct notifier_block *self, - unsigned long action, void *hcpu) -{ - int cpu = (unsigned long)hcpu; - switch(action) { - case CPU_UP_PREPARE: - kstat_init_cpu(cpu); - break; - default: - break; - } - return NOTIFY_OK; -} - -static struct notifier_block __devinitdata kstat_nb = { - .notifier_call = kstat_cpu_notify, - .next = NULL, -}; - -__init static void init_kstat(void) { - kstat_cpu_notify(&kstat_nb, (unsigned long)CPU_UP_PREPARE, - (void *)(long)smp_processor_id()); - register_cpu_notifier(&kstat_nb); -} - -void __init sched_init(void) -{ - runqueue_t *rq; - int i, j, k; - - /* Init the kstat counters */ - init_kstat(); - for (i = 0; i < NR_CPUS; i++) { - prio_array_t *array; - - rq = cpu_rq(i); - rq->active = rq->arrays; - rq->expired = rq->arrays + 1; - spin_lock_init(&rq->lock); - INIT_LIST_HEAD(&rq->migration_queue); - atomic_set(&rq->nr_iowait, 0); - nr_running_init(rq); - - for (j = 0; j < 2; j++) { - array = rq->arrays + j; - for (k = 0; k < MAX_PRIO; k++) { - INIT_LIST_HEAD(array->queue + k); - __clear_bit(k, array->bitmap); - } - // delimiter for bitsearch - __set_bit(MAX_PRIO, array->bitmap); - } - } - /* - * We have to do a little magic to get the first - * thread right in SMP mode. - */ - rq = this_rq(); - rq->curr = current; - rq->idle = current; - set_task_cpu(current, smp_processor_id()); - wake_up_forked_process(current); - - init_timers(); - - /* - * The boot idle thread does lazy MMU switching as well: - */ - atomic_inc(&init_mm.mm_count); - enter_lazy_tlb(&init_mm, current); -} - -#ifdef CONFIG_DEBUG_SPINLOCK_SLEEP -void __might_sleep(char *file, int line) -{ -#if defined(in_atomic) - static unsigned long prev_jiffy; /* ratelimiting */ - - if ((in_atomic() || irqs_disabled()) && system_running) { - if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy) - return; - prev_jiffy = jiffies; - printk(KERN_ERR "Debug: sleeping function called from invalid" - " context at %s:%d\n", file, line); - printk("in_atomic():%d, irqs_disabled():%d\n", - in_atomic(), irqs_disabled()); - dump_stack(); - } -#endif -} -EXPORT_SYMBOL(__might_sleep); -#endif - - -#if defined(CONFIG_SMP) && defined(CONFIG_PREEMPT) -/* - * This could be a long-held lock. If another CPU holds it for a long time, - * and that CPU is not asked to reschedule then *this* CPU will spin on the - * lock for a long time, even if *this* CPU is asked to reschedule. - * - * So what we do here, in the slow (contended) path is to spin on the lock by - * hand while permitting preemption. - * - * Called inside preempt_disable(). - */ -void __preempt_spin_lock(spinlock_t *lock) -{ - if (preempt_count() > 1) { - _raw_spin_lock(lock); - return; - } - do { - preempt_enable(); - while (spin_is_locked(lock)) - cpu_relax(); - preempt_disable(); - } while (!_raw_spin_trylock(lock)); -} - -EXPORT_SYMBOL(__preempt_spin_lock); - -void __preempt_write_lock(rwlock_t *lock) -{ - if (preempt_count() > 1) { - _raw_write_lock(lock); - return; - } - - do { - preempt_enable(); - while (rwlock_is_locked(lock)) - cpu_relax(); - preempt_disable(); - } while (!_raw_write_trylock(lock)); -} - -EXPORT_SYMBOL(__preempt_write_lock); -#endif /* defined(CONFIG_SMP) && defined(CONFIG_PREEMPT) */